diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java b/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java index ed327f70d1a..e526851c653 100644 --- a/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java @@ -148,22 +148,47 @@ public static void setTerm(IteratorSetting cfg, String term) { cfg.addOption(TERM_OPT, term); } + /** + * Enable or disable matching on the row field of the key. Defaults to enable. + * + * @since 2.1.3 + */ public static void matchRow(IteratorSetting cfg, boolean match) { cfg.addOption(MATCH_ROW_OPT, Boolean.toString(match)); } + /** + * Enable or disable matching on the family field of the key. Defaults to enable. + * + * @since 2.1.3 + */ public static void matchColumnFamily(IteratorSetting cfg, boolean match) { cfg.addOption(MATCH_COLFAM_OPT, Boolean.toString(match)); } + /** + * Enable or disable matching on the qualifier field of the key. Defaults to enable. + * + * @since 2.1.3 + */ public static void matchColumnQualifier(IteratorSetting cfg, boolean match) { cfg.addOption(MATCH_COLQUAL_OPT, Boolean.toString(match)); } + /** + * Enable or disable matching on the visibility field of the key. Defaults to disable. + * + * @since 2.1.3 + */ public static void matchColumnVisibility(IteratorSetting cfg, boolean match) { cfg.addOption(MATCH_COLVIS_OPT, Boolean.toString(match)); } + /** + * Enable or disable matching on the value. Defaults to enable. + * + * @since 2.1.3 + */ public static void matchValue(IteratorSetting cfg, boolean match) { cfg.addOption(MATCH_VALUE_OPT, Boolean.toString(match)); } diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/system/VisibilityFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/system/VisibilityFilterTest.java index e12e95b0466..76dc8a01eb5 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/system/VisibilityFilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/system/VisibilityFilterTest.java @@ -18,13 +18,16 @@ */ package org.apache.accumulo.core.iterators.system; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; @@ -57,16 +60,97 @@ public void testEmptyAuths() throws IOException { tm.put(new Key("r1", "cf1", "cq1", ""), new Value()); tm.put(new Key("r1", "cf1", "cq2", "C"), new Value()); tm.put(new Key("r1", "cf1", "cq3", ""), new Value()); + tm.put(new Key("r2", "cf1", "cq2", "C"), new Value()); SortedKeyValueIterator filter = - VisibilityFilter.wrap(new SortedMapIterator(tm), Authorizations.EMPTY, "".getBytes()); + VisibilityFilter.wrap(new SortedMapIterator(tm), Authorizations.EMPTY, "".getBytes(UTF_8)); - filter.seek(new Range(), new HashSet<>(), false); - assertTrue(filter.hasTop()); - assertEquals(new Key("r1", "cf1", "cq1", ""), filter.getTopKey()); - filter.next(); - assertTrue(filter.hasTop()); - assertEquals(new Key("r1", "cf1", "cq3", ""), filter.getTopKey()); - filter.next(); - assertFalse(filter.hasTop()); + TreeSet expected = new TreeSet<>(); + expected.add(new Key("r1", "cf1", "cq1", "")); + expected.add(new Key("r1", "cf1", "cq3", "")); + + verify(expected, filter); + } + + @Test + public void testAuths() throws IOException { + TreeMap tm = new TreeMap<>(); + + // want to have repeated col vis in order to exercise cache in test + tm.put(new Key("r1", "cf1", "cq1", "A&B"), new Value()); + tm.put(new Key("r1", "cf1", "cq2", "A|C"), new Value()); + tm.put(new Key("r1", "cf1", "cq3", "A&B"), new Value()); + tm.put(new Key("r2", "cf1", "cq2", "A|C"), new Value()); + tm.put(new Key("r2", "cf1", "cq3", ""), new Value()); + tm.put(new Key("r2", "cf1", "cq2", "C|D"), new Value()); + tm.put(new Key("r3", "cf1", "cq2", "C|(A&D)"), new Value()); + tm.put(new Key("r4", "cf1", "cq2", "C|D"), new Value()); + tm.put(new Key("r5", "cf1", "cq2", "A&B"), new Value()); + tm.put(new Key("r5", "cf1", "cq3", ""), new Value()); + tm.put(new Key("r6", "cf1", "cq2", "C|(A&D)"), new Value()); + + SortedKeyValueIterator filter = VisibilityFilter.wrap(new SortedMapIterator(tm), + new Authorizations("A", "B"), "".getBytes(UTF_8)); + + TreeSet expected = new TreeSet<>(); + expected.add(new Key("r1", "cf1", "cq1", "A&B")); + expected.add(new Key("r1", "cf1", "cq2", "A|C")); + expected.add(new Key("r1", "cf1", "cq3", "A&B")); + expected.add(new Key("r2", "cf1", "cq2", "A|C")); + expected.add(new Key("r5", "cf1", "cq2", "A&B")); + expected.add(new Key("r2", "cf1", "cq3", "")); + expected.add(new Key("r5", "cf1", "cq3", "")); + + verify(expected, filter); + } + + private static void verify(TreeSet expected, SortedKeyValueIterator iter) + throws IOException { + for (var filter : List.of(iter, iter.deepCopy(null))) { + filter.seek(new Range(), Set.of(), false); + var eiter = expected.iterator(); + while (eiter.hasNext() && filter.hasTop()) { + Key ekey = eiter.next(); + assertEquals(ekey, filter.getTopKey()); + filter.next(); + } + + assertFalse(filter.hasTop()); + assertFalse(eiter.hasNext()); + } } + + @Test + public void testDefaultVisibility() throws IOException { + // Test non empty default visibility + var defaultVis = "A&B&C".getBytes(UTF_8); + + TreeMap tm = new TreeMap<>(); + + tm.put(new Key("r1", "cf1", "cq1", "A&B"), new Value()); + tm.put(new Key("r1", "cf1", "cq2", ""), new Value()); + tm.put(new Key("r1", "cf1", "cq3", "A&B"), new Value()); + tm.put(new Key("r1", "cf1", "cq4", ""), new Value()); + // add something that has the same col vis as the defaultVis + tm.put(new Key("r1", "cf1", "cq5", "A&B&C"), new Value()); + tm.put(new Key("r1", "cf1", "cq6", ""), new Value()); + tm.put(new Key("r1", "cf1", "cq7", "A&B&C"), new Value()); + + // with the set of auths [A,B] the default visibility is not visible + SortedKeyValueIterator filter = + VisibilityFilter.wrap(new SortedMapIterator(tm), new Authorizations("A", "B"), defaultVis); + + TreeSet expected = new TreeSet<>(); + expected.add(new Key("r1", "cf1", "cq1", "A&B")); + expected.add(new Key("r1", "cf1", "cq3", "A&B")); + + verify(expected, filter); + + // with the set of auths [A.B.C] should be able to see all data + filter = VisibilityFilter.wrap(new SortedMapIterator(tm), new Authorizations("A", "B", "C"), + defaultVis); + filter.seek(new Range(), Set.of(), false); + + verify(new TreeSet<>(tm.keySet()), filter); + } + } diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/GrepIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/GrepIteratorTest.java index 191a0403a07..31acabac687 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/GrepIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/GrepIteratorTest.java @@ -68,6 +68,9 @@ public void init() { output.put(new Key("dfh", "xyz", "xyz", 0), new Value("abcdef")); input.put(new Key("dfh", "xyz", "xyz", 1), new Value("xyz")); + input.put(new Key("dfh", "xyz", "xyz", "abcdef", 0), new Value("xyz")); + output.put(new Key("dfh", "xyz", "xyz", "abcdef", 0), new Value("xyz")); + Key k = new Key("dfh", "xyz", "xyz", 1); k.setDeleted(true); input.put(k, new Value("xyz")); @@ -82,6 +85,7 @@ public static void checkEntries(SortedKeyValueIterator skvi, SortedMa assertEquals(e.getValue(), skvi.getTopValue()); skvi.next(); } + assertFalse(skvi.hasTop()); } @@ -90,9 +94,9 @@ public void test() throws IOException { GrepIterator gi = new GrepIterator(); IteratorSetting is = new IteratorSetting(1, GrepIterator.class); GrepIterator.setTerm(is, "ab"); + GrepIterator.matchColumnVisibility(is, true); gi.init(new SortedMapIterator(input), is.getOptions(), null); gi.seek(new Range(), EMPTY_COL_FAMS, false); - GrepIterator.matchColumnVisibility(is, true); checkEntries(gi, output); GrepIterator.setTerm(is, "cde"); gi.init(new SortedMapIterator(input), is.getOptions(), null); @@ -118,4 +122,139 @@ public void test() throws IOException { checkEntries(gi, output); } + + @Test + public void testMatchRow() throws Exception { + GrepIterator gi = new GrepIterator(); + IteratorSetting is = new IteratorSetting(1, GrepIterator.class); + + GrepIterator.setTerm(is, "abcdef"); + + GrepIterator.matchRow(is, true); + GrepIterator.matchColumnFamily(is, false); + GrepIterator.matchColumnQualifier(is, false); + GrepIterator.matchColumnVisibility(is, false); + GrepIterator.matchValue(is, false); + + gi.init(new SortedMapIterator(input), is.getOptions(), null); + gi.seek(new Range(), EMPTY_COL_FAMS, false); + + SortedMap expectedOutput = new TreeMap<>(); + input.forEach((k, v) -> { + if (k.getRowData().toString().contains("abcdef") || k.isDeleted()) { + expectedOutput.put(k, v); + } + }); + + assertFalse(expectedOutput.isEmpty()); + checkEntries(gi, expectedOutput); + } + + @Test + public void testMatchFamily() throws Exception { + GrepIterator gi = new GrepIterator(); + IteratorSetting is = new IteratorSetting(1, GrepIterator.class); + + GrepIterator.setTerm(is, "abcdef"); + + GrepIterator.matchRow(is, false); + GrepIterator.matchColumnFamily(is, true); + GrepIterator.matchColumnQualifier(is, false); + GrepIterator.matchColumnVisibility(is, false); + GrepIterator.matchValue(is, false); + + gi.init(new SortedMapIterator(input), is.getOptions(), null); + gi.seek(new Range(), EMPTY_COL_FAMS, false); + + SortedMap expectedOutput = new TreeMap<>(); + input.forEach((k, v) -> { + if (k.getColumnFamilyData().toString().contains("abcdef") || k.isDeleted()) { + expectedOutput.put(k, v); + } + }); + + assertFalse(expectedOutput.isEmpty()); + checkEntries(gi, expectedOutput); + } + + @Test + public void testMatchQualifier() throws Exception { + GrepIterator gi = new GrepIterator(); + IteratorSetting is = new IteratorSetting(1, GrepIterator.class); + + GrepIterator.setTerm(is, "abcdef"); + + GrepIterator.matchRow(is, false); + GrepIterator.matchColumnFamily(is, false); + GrepIterator.matchColumnQualifier(is, true); + GrepIterator.matchColumnVisibility(is, false); + GrepIterator.matchValue(is, false); + + gi.init(new SortedMapIterator(input), is.getOptions(), null); + gi.seek(new Range(), EMPTY_COL_FAMS, false); + + SortedMap expectedOutput = new TreeMap<>(); + input.forEach((k, v) -> { + if (k.getColumnQualifierData().toString().contains("abcdef") || k.isDeleted()) { + expectedOutput.put(k, v); + } + }); + + assertFalse(expectedOutput.isEmpty()); + checkEntries(gi, expectedOutput); + } + + @Test + public void testMatchVisibility() throws Exception { + GrepIterator gi = new GrepIterator(); + IteratorSetting is = new IteratorSetting(1, GrepIterator.class); + + GrepIterator.setTerm(is, "abcdef"); + + GrepIterator.matchRow(is, false); + GrepIterator.matchColumnFamily(is, false); + GrepIterator.matchColumnQualifier(is, false); + GrepIterator.matchColumnVisibility(is, true); + GrepIterator.matchValue(is, false); + + gi.init(new SortedMapIterator(input), is.getOptions(), null); + gi.seek(new Range(), EMPTY_COL_FAMS, false); + + SortedMap expectedOutput = new TreeMap<>(); + input.forEach((k, v) -> { + if (k.getColumnVisibilityData().toString().contains("abcdef") || k.isDeleted()) { + expectedOutput.put(k, v); + } + }); + + assertFalse(expectedOutput.isEmpty()); + checkEntries(gi, expectedOutput); + } + + @Test + public void testMatchValue() throws Exception { + GrepIterator gi = new GrepIterator(); + IteratorSetting is = new IteratorSetting(1, GrepIterator.class); + + GrepIterator.setTerm(is, "abcdef"); + + GrepIterator.matchRow(is, false); + GrepIterator.matchColumnFamily(is, false); + GrepIterator.matchColumnQualifier(is, false); + GrepIterator.matchColumnVisibility(is, false); + GrepIterator.matchValue(is, true); + + gi.init(new SortedMapIterator(input), is.getOptions(), null); + gi.seek(new Range(), EMPTY_COL_FAMS, false); + + SortedMap expectedOutput = new TreeMap<>(); + input.forEach((k, v) -> { + if (v.toString().contains("abcdef") || k.isDeleted()) { + expectedOutput.put(k, v); + } + }); + + assertFalse(expectedOutput.isEmpty()); + checkEntries(gi, expectedOutput); + } }