Skip to content

GH-4362: Make OpDatasetNames work on in-memory and wrapped TDB2 datasets #4387

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
krickert wants to merge 1 commit into
base: main
Choose a base branch
from
Open

GH-4362: Make OpDatasetNames work on in-memory and wrapped TDB2 datasets #4387

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 25 additions & 7 deletions jena-arq/src/main/java/org/apache/jena/sparql/engine/main/OpExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@
import org.apache.jena.query.ARQ;
import org.apache.jena.query.QueryExecException;
import org.apache.jena.query.SortCondition;
import org.apache.jena.sparql.ARQNotImplemented;
import org.apache.jena.sparql.algebra.Op;
import org.apache.jena.sparql.algebra.OpVars;
import org.apache.jena.sparql.algebra.op.*;
import org.apache.jena.sparql.core.BasicPattern;
import org.apache.jena.sparql.core.DatasetGraph;
import org.apache.jena.sparql.core.Quad;
import org.apache.jena.sparql.core.Var;
import org.apache.jena.sparql.engine.ExecutionContext;
Expand All @@ -48,6 +48,7 @@
import org.apache.jena.sparql.engine.main.iterator.QueryIterUnion;
import org.apache.jena.sparql.expr.Expr;
import org.apache.jena.sparql.expr.ExprList;
import org.apache.jena.sparql.graph.GraphOps;
import org.apache.jena.sparql.procedure.ProcEval;
import org.apache.jena.sparql.procedure.Procedure;
import org.apache.jena.sparql.service.ServiceExec;
Expand Down Expand Up @@ -322,14 +323,31 @@ protected QueryIterator execute(OpService opService, QueryIterator input) {
return ServiceExec.exec(opService, input, execCxt);
}

// Quad form, "GRAPH ?g {}" Flip back to OpGraph.
// Normally quad stores override this.
/**
* Evaluate {@link OpDatasetNames} (quad form of {@code GRAPH ?g {}} or {@code GRAPH <iri> {}})
* against the active dataset in {@link #execCxt}.
*/
protected QueryIterator execute(OpDatasetNames dsNames, QueryIterator input) {
if ( false ) {
OpGraph op = new OpGraph(dsNames.getGraphNode(), new OpBGP());
return execute(op, input);
DatasetGraph ds = execCxt.getDataset();
Node graphNode = dsNames.getGraphNode();
if ( !Var.isVar(graphNode) ) {
if ( (graphNode.isURI() || graphNode.isBlank()) && GraphOps.containsGraph(ds, graphNode) )
return input;
input.close();
return QueryIterNullIterator.create(execCxt);
}
throw new ARQNotImplemented("execute/OpDatasetNames");
Var var = Var.alloc(graphNode);
return new QueryIterRepeatApply(input, execCxt) {
@Override
protected QueryIterator nextStage(Binding binding) {
Node gn = Var.lookup(binding, var);
if ( Var.isVar(gn) )
return new QueryIterExtendByVar(binding, var, ds.listGraphNodes(), getExecContext());
if ( (gn.isURI() || gn.isBlank()) && GraphOps.containsGraph(ds, gn) )
return QueryIterSingleton.create(binding, getExecContext());
return QueryIterNullIterator.create(getExecContext());
}
};
}

protected QueryIterator execute(OpTable opTable, QueryIterator input) {
Expand Down
1 change: 1 addition & 0 deletions jena-arq/src/test/java/org/apache/jena/sparql/engine/TS_Engine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
TestQueryEngineMultiThreaded.class
, TestQueryEngineFromContext.class
, TestQueryEngine.class
, TestOpDatasetNames.class
, TestJsonEval.class
})

Expand Down
203 changes: 203 additions & 0 deletions jena-arq/src/test/java/org/apache/jena/sparql/engine/TestOpDatasetNames.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,203 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/

package org.apache.jena.sparql.engine;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.util.List;
import java.util.Set;
import java.util.function.Predicate;

import org.junit.jupiter.api.Test;

import org.apache.jena.atlas.iterator.Iter;
import org.apache.jena.atlas.lib.SetUtils;
import org.apache.jena.graph.Node;
import org.apache.jena.graph.NodeFactory;
import org.apache.jena.sparql.core.DatasetGraph;
import org.apache.jena.sparql.core.DatasetGraphFactory;
import org.apache.jena.sparql.core.DatasetGraphFilteredView;
import org.apache.jena.sparql.core.DatasetGraphWrapper;
import org.apache.jena.sparql.core.Quad;
import org.apache.jena.sparql.engine.binding.Binding;
import org.apache.jena.sparql.engine.main.QueryEngineMainQuad;
import org.apache.jena.sparql.exec.QueryExec;
import org.apache.jena.sparql.sse.SSE;
import org.apache.jena.sparql.util.Context;
import org.apache.jena.system.Txn;

/**
* Tests for {@code OpDatasetNames} evaluation via {@link QueryEngineMainQuad}.
*/
public class TestOpDatasetNames {

private static final String NS = "http://example/";
private static final Node G1 = NodeFactory.createURI(NS + "g1");
private static final Node G2 = NodeFactory.createURI(NS + "g2");
private static final Quad Q_DFT_G1 = SSE.parseQuad("(_ <http://example/s> <http://example/p> <http://example/g1>)");
private static final Quad Q_DFT_NONE = SSE.parseQuad("(_ <http://example/s> <http://example/p> <http://example/nonexistent>)");
private static final Quad Q_G1 = SSE.parseQuad("(<http://example/g1> <http://example/s1> <http://example/p> 1)");
private static final Quad Q_G2 = SSE.parseQuad("(<http://example/g2> <http://example/s2> <http://example/p> 2)");

private static final String QUERY_UNBOUND = "SELECT ?g { GRAPH ?g {} }";
private static final String QUERY_BOUND_G1 = "SELECT ?g { VALUES ?g { <http://example/g1> } GRAPH ?g {} }";
private static final String QUERY_BOUND_NONEXISTENT = "SELECT ?g { VALUES ?g { <http://example/nonexistent> } GRAPH ?g {} }";
private static final String QUERY_BOUND_LITERAL = "SELECT ?g { VALUES ?g { 'not-a-graph' } GRAPH ?g {} }";
private static final String QUERY_SEQUENCE_BOUND = "SELECT ?g { <http://example/s> <http://example/p> ?g . GRAPH ?g {} }";
private static final String QUERY_SEQUENCE_INPUT = "SELECT ?s ?g { <http://example/s> <http://example/p> ?s . GRAPH ?g {} }";
private static final String ASK_G1 = "ASK { GRAPH <http://example/g1> {} }";
private static final String ASK_NONEXISTENT = "ASK { GRAPH <http://example/nonexistent> {} }";
private static final String ASK_DEFAULT_GRAPH = "ASK { GRAPH <urn:x-arq:DefaultGraph> {} }";
private static final String ASK_UNION_GRAPH = "ASK { GRAPH <urn:x-arq:UnionGraph> {} }";

@Test
public void opDatasetNames_emptyDataset() {
DatasetGraph dsg = DatasetGraphFactory.createTxnMem();
Txn.executeRead(dsg, () -> {
Set<Node> graphs = selectGraphsQuadEngine(dsg, QUERY_UNBOUND);
assertTrue(graphs.isEmpty());
assertFalse(askQuadEngine(dsg, ASK_G1));
});
}

@Test
public void opDatasetNames_unboundVar() {
DatasetGraph dsg = createPopulatedTxnMem();
Txn.executeRead(dsg, () -> {
Set<Node> graphs = selectGraphsQuadEngine(dsg, QUERY_UNBOUND);
assertEquals(Set.of(G1, G2), graphs);
});
}

@Test
public void opDatasetNames_preBoundMatchingVar() {
DatasetGraph dsg = createPopulatedTxnMem();
Txn.executeRead(dsg, () -> {
Set<Node> graphs = selectGraphsQuadEngine(dsg, QUERY_BOUND_G1);
assertEquals(Set.of(G1), graphs);
});
}

@Test
public void opDatasetNames_preBoundNonMatchingVar() {
DatasetGraph dsg = createPopulatedTxnMem();
Txn.executeRead(dsg, () -> {
assertTrue(selectGraphsQuadEngine(dsg, QUERY_BOUND_NONEXISTENT).isEmpty());
assertTrue(selectGraphsQuadEngine(dsg, QUERY_BOUND_LITERAL).isEmpty());
});
}

@Test
public void opDatasetNames_sequenceBoundVar() {
DatasetGraph dsg = createPopulatedTxnMem();
Txn.executeRead(dsg, () -> {
Set<Node> graphs = selectGraphsQuadEngine(dsg, QUERY_SEQUENCE_BOUND);
assertEquals(Set.of(G1), graphs);
});
}

@Test
public void opDatasetNames_concreteGraphName() {
DatasetGraph dsg = createPopulatedTxnMem();
Txn.executeRead(dsg, () -> {
assertTrue(askQuadEngine(dsg, ASK_G1));
assertFalse(askQuadEngine(dsg, ASK_NONEXISTENT));
assertTrue(askQuadEngine(dsg, ASK_DEFAULT_GRAPH));
assertTrue(askQuadEngine(dsg, ASK_UNION_GRAPH));
});
}

@Test
public void opDatasetNames_wrappedAndFilteredView() {
DatasetGraph base = createPopulatedTxnMem();
Txn.executeRead(base, () -> {
DatasetGraph wrapped = new DatasetGraphWrapper(base);
assertEquals(Set.of(G1, G2), selectGraphsQuadEngine(wrapped, QUERY_UNBOUND));
assertEquals(Set.of(G1), selectGraphsQuadEngine(wrapped, QUERY_BOUND_G1));
assertTrue(selectGraphsQuadEngine(wrapped, QUERY_BOUND_NONEXISTENT).isEmpty());
assertTrue(askQuadEngine(wrapped, ASK_G1));
assertFalse(askQuadEngine(wrapped, ASK_NONEXISTENT));

Predicate<Quad> filter = q -> q.getGraph().equals(G1);
DatasetGraph filtered = new DatasetGraphFilteredView(base, filter, Set.of(G1));
assertEquals(Set.of(G1), selectGraphsQuadEngine(filtered, QUERY_UNBOUND));
assertEquals(Set.of(G1), selectGraphsQuadEngine(filtered, QUERY_BOUND_G1));
assertTrue(selectGraphsQuadEngine(filtered, QUERY_BOUND_NONEXISTENT).isEmpty());
assertTrue(askQuadEngine(filtered, ASK_G1));
assertFalse(askQuadEngine(filtered, ASK_NONEXISTENT));
});
}

@Test
public void opDatasetNames_preservesInputBindings() {
DatasetGraph dsg = createPopulatedTxnMem();
Txn.executeRead(dsg, () -> {
List<Binding> rows = selectBindingsQuadEngine(dsg, QUERY_SEQUENCE_INPUT);
assertEquals(4, rows.size());
Set<String> pairs = SetUtils.toSet(rows.stream().map(b -> b.get("s").getURI() + " " + b.get("g").getURI()));
assertEquals(Set.of(NS + "g1 " + NS + "g1",
NS + "g1 " + NS + "g2",
NS + "nonexistent " + NS + "g1",
NS + "nonexistent " + NS + "g2"), pairs);
});
}

private DatasetGraph createPopulatedTxnMem() {
DatasetGraph dsg = DatasetGraphFactory.createTxnMem();
Txn.executeWrite(dsg, () -> {
dsg.add(Q_DFT_G1);
dsg.add(Q_DFT_NONE);
dsg.add(Q_G1);
dsg.add(Q_G2);
});
return dsg;
}

private Set<Node> selectGraphsQuadEngine(DatasetGraph dsg, String queryString) {
List<Binding> rows = selectBindingsQuadEngine(dsg, queryString);
return SetUtils.toSet(rows.stream().map(b -> b.get("g")));
}

private List<Binding> selectBindingsQuadEngine(DatasetGraph dsg, String queryString) {
Context cxt = quadEngineContext(dsg);
try ( QueryExec qExec = QueryExec.dataset(dsg).query(queryString).context(cxt).build() ) {
return Iter.toList(qExec.select());
}
}

private boolean askQuadEngine(DatasetGraph dsg, String queryString) {
Context cxt = quadEngineContext(dsg);
try ( QueryExec qExec = QueryExec.dataset(dsg).query(queryString).context(cxt).build() ) {
return qExec.ask();
}
}

private Context quadEngineContext(DatasetGraph dsg) {
Context cxt = Context.setupContextForDataset(null, dsg);
QueryEngineRegistry reg = new QueryEngineRegistry();
reg.add(QueryEngineMainQuad.getFactory());
QueryEngineRegistry.set(cxt, reg);
return cxt;
}
}
10 changes: 9 additions & 1 deletion jena-tdb2/src/main/java/org/apache/jena/tdb2/solver/OpExecutorTDB2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import org.apache.jena.tdb2.store.DatasetGraphTDB;
import org.apache.jena.tdb2.store.GraphTDB;
import org.apache.jena.tdb2.store.NodeId;
import org.apache.jena.tdb2.sys.TDBInternal;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -324,9 +325,16 @@ private static boolean isDefaultGraphStorage(Node gn) {
return false;
}

/**
* Evaluate {@link OpDatasetNames} against the underlying {@link DatasetGraphTDB},
* or delegate to {@link OpExecutor#execute(OpDatasetNames, QueryIterator)} when
* the execution dataset is a non-TDB wrapper or view.
*/
@Override
protected QueryIterator execute(OpDatasetNames dsNames, QueryIterator input) {
DatasetGraphTDB ds = (DatasetGraphTDB)execCxt.getDataset();
DatasetGraphTDB ds = TDBInternal.getDatasetGraphTDB(execCxt.getDataset());
if ( ds == null )
return super.execute(dsNames, input);
Predicate<Tuple<NodeId>> filter = QC2.getFilter(execCxt.getContext());
Node gn = dsNames.getGraphNode();
if ( Var.isVar(gn) )
Expand Down
42 changes: 30 additions & 12 deletions jena-tdb2/src/main/java/org/apache/jena/tdb2/solver/SolverLibTDB.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,17 @@ static BindingNodeId convert(Binding binding, NodeTable nodeTable) {
/** Find whether a specific graph name is in the quads table. */
static QueryIterator testForGraphName(DatasetGraphTDB ds, Node graphNode, QueryIterator input,
Predicate<Tuple<NodeId>> filter, ExecutionContext execCxt) {
boolean exists = hasGraphName(ds, graphNode, filter);
if ( exists )
return input;
else {
input.close();
return QueryIterNullIterator.create(execCxt);
}
}

/** Return {@code true} if {@code graphNode} is present as a graph name in the quads table. */
private static boolean hasGraphName(DatasetGraphTDB ds, Node graphNode, Predicate<Tuple<NodeId>> filter) {
NodeId nid = TDBInternal.getNodeId(ds, graphNode);
boolean exists = !NodeId.isDoesNotExist(nid);
if ( exists ) {
Expand All @@ -139,13 +150,7 @@ static QueryIterator testForGraphName(DatasetGraphTDB ds, Node graphNode, QueryI
iter1 = Iter.filter(iter1, filter);
exists = iter1.hasNext();
}

if ( exists )
return input;
else {
input.close();
return QueryIterNullIterator.create(execCxt);
}
return exists;
}

private static Tuple<NodeId> TupleANY = TupleFactory.create4(NodeId.NodeIdAny, NodeId.NodeIdAny, NodeId.NodeIdAny, NodeId.NodeIdAny);
Expand All @@ -158,6 +163,23 @@ private enum GraphNamesDistinctMode {
static QueryIterator graphNames(DatasetGraphTDB ds, Node graphNode, QueryIterator input,
Predicate<Tuple<NodeId>> filter, ExecutionContext execCxt) {
List<Abortable> killList = new ArrayList<>();
final Var var = Var.alloc(graphNode);
Iterator<Binding> iterBinding = Iter.flatMap(input, binding -> {
Node gn = Var.lookup(binding, var);
if ( !Var.isVar(gn) ) {
if ( hasGraphName(ds, gn, filter) )
return Iter.singletonIterator(binding);
return Iter.nullIterator();
}
Iterator<Node> iterNodes = listGraphNodes(ds, killList, filter, execCxt);
return Iter.map(iterNodes, node -> BindingFactory.binding(binding, var, node));
});
return new QueryIterAbortable(iterBinding, killList, input, execCxt);
}

/** Return an iterator of distinct graph nodes from the quads table. */
private static Iterator<Node> listGraphNodes(DatasetGraphTDB ds, List<Abortable> killList,
Predicate<Tuple<NodeId>> filter, ExecutionContext execCxt) {
NodeTupleTable ntt = ds.getQuadTable().getNodeTupleTable();

// Select a graph based index for use, if a suitable one is available
Expand Down Expand Up @@ -211,11 +233,7 @@ static QueryIterator graphNames(DatasetGraphTDB ds, Node graphNode, QueryIterato
default -> iter2;
};

Iterator<Node> iter4 = NodeLib.nodes(ds.getQuadTable().getNodeTupleTable().getNodeTable(), iter3);

final Var var = Var.alloc(graphNode);
Iterator<Binding> iterBinding = Iter.map(iter4, node -> BindingFactory.binding(var, node));
return new QueryIterAbortable(iterBinding, killList, input, execCxt);
return NodeLib.nodes(ds.getQuadTable().getNodeTupleTable().getNodeTable(), iter3);
}

static Set<NodeId> convertToNodeIds(Collection<Node> nodes, DatasetGraphTDB dataset)
Expand Down
Loading
Loading