Python ferenda.TripleStore类代码示例

    def temp_analyze(self):
        store = TripleStore(self.config.storetype,
                            self.config.storelocation,
                            self.config.storerepository)
        # sq = self._query_cites('http://lagen.nu/ext/celex/12008E045',self._sameas(),False, True, 2012)
        sq = self._query_cites(None, self._sameas(), False, False, 2012)
        print(sq)
        cites = store.select(sq, format="python")
        self.log.debug(
            "    Citation graph contains %s citations" % (len(cites)))

        # remove duplicate citations, self-citations and pinpoints
        # in citations
        citedict = {}
        for cite in cites:
            # print repr(cite)
            if "-" in cite['obj']:
                cite['obj'] = cite['obj'].split("-")[0]

            if (cite['subj'] != cite['obj']):
                citedict[(cite['subj'], cite['obj'])] = True

        self.log.debug(
            "    Normalized graph contains %s citations" % len(citedict))

        degree = {}
        for citing, cited in list(citedict.keys()):
            if citing not in degree:
                degree[citing] = []
            if cited not in degree:
                degree[cited] = []
            degree[cited].append(citing)

        return

 def tearDown(self):
     self.setupclass = False # make sure super.tearDown deletes all files
     super(AdvancedAPI, self).tearDown()
     FulltextIndex.connect(self.indextype, self.indexlocation,
                           [DocumentRepository()]).destroy()
     TripleStore.connect(self.storetype, self.storelocation,
                         self.storerepository).clear()

 def download_from_triplestore(self):
     sq = "SELECT ?something ?references ?uri where ?something ?references ?uri AND NOT ?uri ?references ?anything"
     store = TripleStore(self.config.storetype,
                         self.config.storelocation,
                         self.config.storerepository)
     with self.store.open_downloaded("biggraph") as fp:
         for row in store.select(sq):
             fp.write("<%(something)s> <%(references)s> <%(uri)s> .\n")

    def test_sqlite_add_serialized(self, mock_graph):
        store = TripleStore.connect("SQLITE", "", "")
        store.add_serialized("tripledata", "nt")
        self.assertTrue(mock_graph.return_value.parse.called)
        self.assertTrue(mock_graph.return_value.commit.called)
        mock_graph.reset_mock()
        
        store.add_serialized("tripledata", "nt", "namedgraph")
        self.assertTrue(mock_graph.return_value.get_context.called)
        self.assertTrue(mock_graph.return_value.get_context.return_value.parse.called)

        store = TripleStore.connect("SQLITE", "", "", inmemory=True)
        with self.assertRaises(errors.TriplestoreError):
            store.add_serialized("tripledata", "nt")

 def test_fuseki_get_serialized(self, mock_get):
     store = TripleStore.connect("FUSEKI", "", "", curl=False)
     # test 1: a namedgraph (cases with no context are already run by
     # test_fuseki_get_serialized_file)
     want = util.readfile("test/files/triplestore/namedgraph.nt", "rb")
     got = store.get_serialized(context="namedgraph") # results in single get
     self.assertEqual(want, got)

 def test_sqlite_add_serialized_file(self, mock_graph):
     store = TripleStore.connect("SQLITE", "", "")
     fd, tmpname = mkstemp()
     fp = os.fdopen(fd, "w")
     fp.write("tripledata")
     fp.close()
     store.add_serialized_file(tmpname, "nt")
     os.unlink(tmpname)

 def test_sqlite_clear(self, mock_graph):
     store = TripleStore.connect("SQLITE", "", "")
     g = Graph()
     g.add((URIRef("http://example.org/doc1"), RDFS.comment, Literal("Hey")))
     g.add((URIRef("http://example.org/doc2"), RDFS.comment, Literal("Ho")))
     mock_graph.return_value.get_context.return_value = g
     store.clear("namedgraph")
     self.assertEqual(2, mock_graph.return_value.remove.call_count)
     self.assertEqual(1, mock_graph.return_value.commit.call_count)

    def test_sqlite_init(self, mock_graph):
        # create a new db that doesnt exist
        mock_graph.open.return_value = 42
        store = TripleStore.connect("SQLITE", "", "")
        self.assertTrue(mock_graph.return_value.open.called)
        self.assertTrue(mock_graph.return_value.open.call_args[1]['create'])

        # reopen an existing db
        fd, tmpname = mkstemp()
        fp = os.fdopen(fd)
        fp.close()
        store = TripleStore.connect("SQLITE", tmpname, "")
        os.unlink(tmpname)
        self.assertFalse(mock_graph.return_value.open.call_args[1]['create'])

        # make an inmemory db
        store = TripleStore.connect("SQLITE", "", "", inmemory=True)
        self.assertTrue(mock_graph.return_value.quads.called)
        self.assertTrue(mock_graph.return_value.addN.called)

    def download(self, basefile=None):
        # Get all "term sets" (used dcterms:subject Objects, wiki pages
        # describing legal concepts, swedish wikipedia pages...)
        terms = defaultdict(dict)

        # 1) Query the triplestore for all dcterms:subject triples (is this
        # semantically sensible for a "download" action -- the content
        # isn't really external?) -- term set "subjects" (these come
        # from both court cases and legal definitions in law text)
        sq = """
        PREFIX dcterms:<http://purl.org/dc/terms/>
        PREFIX rdfs:<http://www.w3.org/2000/01/rdf-schema#>

        SELECT ?uri ?subject ?label
        WHERE { {?uri dcterms:subject ?subject . }
                OPTIONAL {?subject rdfs:label ?label . } }
        """
        store = TripleStore.connect(self.config.storetype,
                                    self.config.storelocation,
                                    self.config.storerepository)
        results = store.select(sq, "python")
        for row in results:
            if 'label' in row:
                label = row['label']
            else:
                label = self.basefile_from_uri(row['subject'])
                if label is None:
                    self.log.warning("could not determine keyword from %s" % row['subject'])
                    continue
            
            sanitized = self.sanitize_term(label)
            if sanitized:
                if sanitized not in terms:
                    terms[sanitized]['subjects'] = []
                terms[sanitized]['subjects'].append(row['uri'])

        self.log.debug("Retrieved %s subject terms from triplestore" % len(terms))

        for termset_func in self.termset_funcs:
            termset_func(terms)

        for term in terms:
            term = self.sanitize_term(term)
            if not term:
                continue
            oldterms = ""
            termpath = self.store.downloaded_path(term)
            if os.path.exists(termpath):
                oldterms = yaml.load(util.readfile(termpath))
            if terms[term] != oldterms:
                util.ensure_dir(termpath)
                util.writefile(termpath, yaml.dump(terms[term], default_flow_style=False))
                self.log.info("%s: in %s termsets" % (term, len(terms[term])))
            else:
                self.log.debug("%s: skipped" % term)

    def test_sesame_add_serialized(self, mock_post):
        store = TripleStore.connect("SESAME", "", "")
        rf = util.readfile
        store.add_serialized(rf("test/files/triplestore/defaultgraph.ttl"),
                             format="turtle")
        self.assertEqual(mock_post.call_count, 1)

        store.add_serialized(rf("test/files/triplestore/namedgraph.nt"),
                             format="nt",
                             context="namedgraph")
        self.assertEqual(mock_post.call_count, 2)

    def test_sesame_get_serialized(self, mock_get):
        store = TripleStore.connect("SESAME", "", "")
        want = util.readfile("test/files/triplestore/combinedgraph.nt", "rb")
        got = store.get_serialized() 
        self.assertEqual(want, got)
        self.assertEqual(mock_get.call_count, 1)

        want = util.readfile("test/files/triplestore/namedgraph.nt", "rb")
        got = store.get_serialized(context="namedgraph") # results in single get
        self.assertEqual(want, got)
        self.assertEqual(mock_get.call_count, 2)

    def test_curl(self, runcmd_mock):
        # needs to test add_serialized, add_serialized_file, get_serialized
        # and get_serialized_file. We'll patch util.runcmd and make sure that
        # the command line is correct. We should also have util.runcmd return
        # a non-zero return code once.
        # our util.runcmd replacement should, for the get_serialized file,
        # create a suitable temp file

        store = TripleStore.connect("FUSEKI", "", "", curl=True)
        # 1. add_serialized
        runcmd_mock.return_value = (0, "", "")
        store.add_serialized("tripledata", "nt")
        cmdline = runcmd_mock.call_args[0][0] # first ordered argument
        # replace the temporary file name
        cmdline = re.sub('"@[^"]+"', '"@tempfile.nt"', cmdline)
        self.assertEqual('curl -X POST --data-binary "@tempfile.nt" --header "Content-Type:application/n-triples;charset=UTF-8" "//data?default"', cmdline)
        runcmd_mock.mock_reset()

        # 2. add_serialized_file
        runcmd_mock.return_value = (0, "", "")
        store.add_serialized_file("tempfile.nt", "nt")
        cmdline = runcmd_mock.call_args[0][0] # first ordered argument
        self.assertEqual('curl -X POST --data-binary "@tempfile.nt" --header "Content-Type:application/n-triples;charset=UTF-8" "//data?default"', cmdline)
        runcmd_mock.mock_reset()

        # 3. get_serialized
        def create_tempfile(*args, **kwargs):
            filename = re.search('-o "([^"]+)"', args[0]).group(1)
            with open(filename, "wb") as fp:
                fp.write("tripledata\n".encode())
            return (0, "", "")
        runcmd_mock.side_effect = create_tempfile
        res = store.get_serialized("nt")
        self.assertEqual(b"tripledata\ntripledata\n", res)
        cmdline = runcmd_mock.call_args[0][0] # first ordered argument
        # replace the temporary file name
        cmdline = re.sub('-o "[^"]+"', '-o "tempfile.nt"', cmdline)
        # FIXME is this really right?
        self.assertEqual('curl -o "tempfile.nt" --header "Accept:application/n-triples" "//data?graph=urn:x-arq:UnionGraph"', cmdline)
        runcmd_mock.side_effect = None
        runcmd_mock.mock_reset()

        # 4. get_serialized_file
        store.get_serialized_file("triples.xml", "xml")
        cmdline = runcmd_mock.call_args[0][0] # first ordered argument
        self.assertEqual('curl -o "triples.xml" --header "Accept:application/rdf+xml" "//data?default"', cmdline)
        runcmd_mock.mock_reset()

        # 5. handle errors
        with self.assertRaises(errors.TriplestoreError):
            runcmd_mock.return_value = (1, "", "Internal error")
            store.get_serialized_file("triples.nt", "nt")

    def test_sesame_construct(self, mock_get):
        store = TripleStore.connect("SESAME", "", "")
        rf = util.readfile
        want = Graph()
        want.parse(data=rf("test/files/triplestore/construct-results.ttl"),
                   format="turtle")
        got = store.construct("the-query")
        self.assertEqualGraphs(want, got)
        self.assertEqual(mock_get.call_count, 1)

        with self.assertRaises(errors.TriplestoreError):
            mock_get.side_effect = requests.exceptions.HTTPError("Server error")
            got = store.construct("the-query")

    def eval_get_goldstandard(self, basefile):
        goldstandard = Graph()
        goldstandard_rdf = util.relpath(
            os.path.dirname(__file__) + "/../res/eut/goldstandard.n3")
        goldstandard.load(goldstandard_rdf, format="n3")

        pred = util.ns['ir'] + 'isRelevantFor'
        res = {}
        store = TripleStore(self.config.storetype,
                            self.config.storelocation,
                            self.config.storerepository)
        sq_templ = """PREFIX eurlex:<http://lagen.nu/eurlex#>
                      SELECT ?party ?casenum ?celexnum WHERE {
                          <%s> eurlex:party ?party ;
                               eurlex:casenum ?casenum ;
                               eurlex:celexnum ?celexnum .
                      }"""

        self.log.debug(
            "Loading gold standard relevance judgments for %s" % basefile)
        for article in self._articles(basefile):
            res[article] = []
            for o in goldstandard.objects(URIRef(article), URIRef(pred)):
                res[article].append(str(o))
                # Make sure the case exists and is the case we're looking for
                sq = sq_templ % str(o)
                parties = store.select(sq, format="python")
                if parties:
                    pass
                    # self.log.debug("   %s: %s (%s)" %
                    #               (parties[0]['celexnum'],
                    #                parties[0]['casenum'],
                    #                " v ".join([x['party'] for x in parties])))
                else:
                    self.log.warning("Can't find %s in triple store!" % o)
            self.log.debug("    Gold standard for %s: %s relevant docs" %
                           (article, len(res[article])))
            res[article].sort()
        return res

 def test_sqlite_construct(self, mock_graph):
     store = TripleStore.connect("SQLITE", "", "")
     sq = """CONSTRUCT ?s ?p ?o WHERE {?o ?p ?s . }"""
     g = Graph()
     g.add((URIRef("http://example.org/doc1"), RDFS.comment, Literal("Hey")))
     g.add((URIRef("http://example.org/doc2"), RDFS.comment, Literal("Ho")))
     res = Mock
     res.graph = g
     mock_graph.return_value.query.return_value = res
     self.assertEqual(g, store.construct(sq))
 
     mock_graph.return_value.query.side_effect = pyparsing.ParseException("Syntax error")
     with self.assertRaises(errors.SparqlError):
         store.construct(sq)