Skip to content

Commit

Permalink
Merge branch '2.1'
Browse files Browse the repository at this point in the history
  • Loading branch information
@keith-turner
keith-turner committed Aug 6, 2024
2 parents 092dacf + ea1558e commit 1e75ed1
Show file tree
Hide file tree
Showing 3 changed files with 259 additions and 11 deletions.
25 changes: 25 additions & 0 deletions core/src/main/java/org/apache/accumulo/core/iterators/user/GrepIterator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,22 +148,47 @@ public static void setTerm(IteratorSetting cfg, String term) {
cfg.addOption(TERM_OPT, term);
}

/**
* Enable or disable matching on the row field of the key. Defaults to enable.
*
* @since 2.1.3
*/
public static void matchRow(IteratorSetting cfg, boolean match) {
cfg.addOption(MATCH_ROW_OPT, Boolean.toString(match));
}

/**
* Enable or disable matching on the family field of the key. Defaults to enable.
*
* @since 2.1.3
*/
public static void matchColumnFamily(IteratorSetting cfg, boolean match) {
cfg.addOption(MATCH_COLFAM_OPT, Boolean.toString(match));
}

/**
* Enable or disable matching on the qualifier field of the key. Defaults to enable.
*
* @since 2.1.3
*/
public static void matchColumnQualifier(IteratorSetting cfg, boolean match) {
cfg.addOption(MATCH_COLQUAL_OPT, Boolean.toString(match));
}

/**
* Enable or disable matching on the visibility field of the key. Defaults to disable.
*
* @since 2.1.3
*/
public static void matchColumnVisibility(IteratorSetting cfg, boolean match) {
cfg.addOption(MATCH_COLVIS_OPT, Boolean.toString(match));
}

/**
* Enable or disable matching on the value. Defaults to enable.
*
* @since 2.1.3
*/
public static void matchValue(IteratorSetting cfg, boolean match) {
cfg.addOption(MATCH_VALUE_OPT, Boolean.toString(match));
}
Expand Down
104 changes: 94 additions & 10 deletions core/src/test/java/org/apache/accumulo/core/iterators/system/VisibilityFilterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,16 @@
*/
package org.apache.accumulo.core.iterators.system;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
Expand Down Expand Up @@ -57,16 +60,97 @@ public void testEmptyAuths() throws IOException {
tm.put(new Key("r1", "cf1", "cq1", ""), new Value());
tm.put(new Key("r1", "cf1", "cq2", "C"), new Value());
tm.put(new Key("r1", "cf1", "cq3", ""), new Value());
tm.put(new Key("r2", "cf1", "cq2", "C"), new Value());
SortedKeyValueIterator<Key,Value> filter =
VisibilityFilter.wrap(new SortedMapIterator(tm), Authorizations.EMPTY, "".getBytes());
VisibilityFilter.wrap(new SortedMapIterator(tm), Authorizations.EMPTY, "".getBytes(UTF_8));

filter.seek(new Range(), new HashSet<>(), false);
assertTrue(filter.hasTop());
assertEquals(new Key("r1", "cf1", "cq1", ""), filter.getTopKey());
filter.next();
assertTrue(filter.hasTop());
assertEquals(new Key("r1", "cf1", "cq3", ""), filter.getTopKey());
filter.next();
assertFalse(filter.hasTop());
TreeSet<Key> expected = new TreeSet<>();
expected.add(new Key("r1", "cf1", "cq1", ""));
expected.add(new Key("r1", "cf1", "cq3", ""));

verify(expected, filter);
}

@Test
public void testAuths() throws IOException {
TreeMap<Key,Value> tm = new TreeMap<>();

// want to have repeated col vis in order to exercise cache in test
tm.put(new Key("r1", "cf1", "cq1", "A&B"), new Value());
tm.put(new Key("r1", "cf1", "cq2", "A|C"), new Value());
tm.put(new Key("r1", "cf1", "cq3", "A&B"), new Value());
tm.put(new Key("r2", "cf1", "cq2", "A|C"), new Value());
tm.put(new Key("r2", "cf1", "cq3", ""), new Value());
tm.put(new Key("r2", "cf1", "cq2", "C|D"), new Value());
tm.put(new Key("r3", "cf1", "cq2", "C|(A&D)"), new Value());
tm.put(new Key("r4", "cf1", "cq2", "C|D"), new Value());
tm.put(new Key("r5", "cf1", "cq2", "A&B"), new Value());
tm.put(new Key("r5", "cf1", "cq3", ""), new Value());
tm.put(new Key("r6", "cf1", "cq2", "C|(A&D)"), new Value());

SortedKeyValueIterator<Key,Value> filter = VisibilityFilter.wrap(new SortedMapIterator(tm),
new Authorizations("A", "B"), "".getBytes(UTF_8));

TreeSet<Key> expected = new TreeSet<>();
expected.add(new Key("r1", "cf1", "cq1", "A&B"));
expected.add(new Key("r1", "cf1", "cq2", "A|C"));
expected.add(new Key("r1", "cf1", "cq3", "A&B"));
expected.add(new Key("r2", "cf1", "cq2", "A|C"));
expected.add(new Key("r5", "cf1", "cq2", "A&B"));
expected.add(new Key("r2", "cf1", "cq3", ""));
expected.add(new Key("r5", "cf1", "cq3", ""));

verify(expected, filter);
}

private static void verify(TreeSet<Key> expected, SortedKeyValueIterator<Key,Value> iter)
throws IOException {
for (var filter : List.of(iter, iter.deepCopy(null))) {
filter.seek(new Range(), Set.of(), false);
var eiter = expected.iterator();
while (eiter.hasNext() && filter.hasTop()) {
Key ekey = eiter.next();
assertEquals(ekey, filter.getTopKey());
filter.next();
}

assertFalse(filter.hasTop());
assertFalse(eiter.hasNext());
}
}

@Test
public void testDefaultVisibility() throws IOException {
// Test non empty default visibility
var defaultVis = "A&B&C".getBytes(UTF_8);

TreeMap<Key,Value> tm = new TreeMap<>();

tm.put(new Key("r1", "cf1", "cq1", "A&B"), new Value());
tm.put(new Key("r1", "cf1", "cq2", ""), new Value());
tm.put(new Key("r1", "cf1", "cq3", "A&B"), new Value());
tm.put(new Key("r1", "cf1", "cq4", ""), new Value());
// add something that has the same col vis as the defaultVis
tm.put(new Key("r1", "cf1", "cq5", "A&B&C"), new Value());
tm.put(new Key("r1", "cf1", "cq6", ""), new Value());
tm.put(new Key("r1", "cf1", "cq7", "A&B&C"), new Value());

// with the set of auths [A,B] the default visibility is not visible
SortedKeyValueIterator<Key,Value> filter =
VisibilityFilter.wrap(new SortedMapIterator(tm), new Authorizations("A", "B"), defaultVis);

TreeSet<Key> expected = new TreeSet<>();
expected.add(new Key("r1", "cf1", "cq1", "A&B"));
expected.add(new Key("r1", "cf1", "cq3", "A&B"));

verify(expected, filter);

// with the set of auths [A.B.C] should be able to see all data
filter = VisibilityFilter.wrap(new SortedMapIterator(tm), new Authorizations("A", "B", "C"),
defaultVis);
filter.seek(new Range(), Set.of(), false);

verify(new TreeSet<>(tm.keySet()), filter);
}

}
141 changes: 140 additions & 1 deletion core/src/test/java/org/apache/accumulo/core/iterators/user/GrepIteratorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ public void init() {
output.put(new Key("dfh", "xyz", "xyz", 0), new Value("abcdef"));
input.put(new Key("dfh", "xyz", "xyz", 1), new Value("xyz"));

input.put(new Key("dfh", "xyz", "xyz", "abcdef", 0), new Value("xyz"));
output.put(new Key("dfh", "xyz", "xyz", "abcdef", 0), new Value("xyz"));

Key k = new Key("dfh", "xyz", "xyz", 1);
k.setDeleted(true);
input.put(k, new Value("xyz"));
Expand All @@ -82,6 +85,7 @@ public static void checkEntries(SortedKeyValueIterator<Key,Value> skvi, SortedMa
assertEquals(e.getValue(), skvi.getTopValue());
skvi.next();
}

assertFalse(skvi.hasTop());
}

Expand All @@ -90,9 +94,9 @@ public void test() throws IOException {
GrepIterator gi = new GrepIterator();
IteratorSetting is = new IteratorSetting(1, GrepIterator.class);
GrepIterator.setTerm(is, "ab");
GrepIterator.matchColumnVisibility(is, true);
gi.init(new SortedMapIterator(input), is.getOptions(), null);
gi.seek(new Range(), EMPTY_COL_FAMS, false);
GrepIterator.matchColumnVisibility(is, true);
checkEntries(gi, output);
GrepIterator.setTerm(is, "cde");
gi.init(new SortedMapIterator(input), is.getOptions(), null);
Expand All @@ -118,4 +122,139 @@ public void test() throws IOException {

checkEntries(gi, output);
}

@Test
public void testMatchRow() throws Exception {
GrepIterator gi = new GrepIterator();
IteratorSetting is = new IteratorSetting(1, GrepIterator.class);

GrepIterator.setTerm(is, "abcdef");

GrepIterator.matchRow(is, true);
GrepIterator.matchColumnFamily(is, false);
GrepIterator.matchColumnQualifier(is, false);
GrepIterator.matchColumnVisibility(is, false);
GrepIterator.matchValue(is, false);

gi.init(new SortedMapIterator(input), is.getOptions(), null);
gi.seek(new Range(), EMPTY_COL_FAMS, false);

SortedMap<Key,Value> expectedOutput = new TreeMap<>();
input.forEach((k, v) -> {
if (k.getRowData().toString().contains("abcdef") || k.isDeleted()) {
expectedOutput.put(k, v);
}
});

assertFalse(expectedOutput.isEmpty());
checkEntries(gi, expectedOutput);
}

@Test
public void testMatchFamily() throws Exception {
GrepIterator gi = new GrepIterator();
IteratorSetting is = new IteratorSetting(1, GrepIterator.class);

GrepIterator.setTerm(is, "abcdef");

GrepIterator.matchRow(is, false);
GrepIterator.matchColumnFamily(is, true);
GrepIterator.matchColumnQualifier(is, false);
GrepIterator.matchColumnVisibility(is, false);
GrepIterator.matchValue(is, false);

gi.init(new SortedMapIterator(input), is.getOptions(), null);
gi.seek(new Range(), EMPTY_COL_FAMS, false);

SortedMap<Key,Value> expectedOutput = new TreeMap<>();
input.forEach((k, v) -> {
if (k.getColumnFamilyData().toString().contains("abcdef") || k.isDeleted()) {
expectedOutput.put(k, v);
}
});

assertFalse(expectedOutput.isEmpty());
checkEntries(gi, expectedOutput);
}

@Test
public void testMatchQualifier() throws Exception {
GrepIterator gi = new GrepIterator();
IteratorSetting is = new IteratorSetting(1, GrepIterator.class);

GrepIterator.setTerm(is, "abcdef");

GrepIterator.matchRow(is, false);
GrepIterator.matchColumnFamily(is, false);
GrepIterator.matchColumnQualifier(is, true);
GrepIterator.matchColumnVisibility(is, false);
GrepIterator.matchValue(is, false);

gi.init(new SortedMapIterator(input), is.getOptions(), null);
gi.seek(new Range(), EMPTY_COL_FAMS, false);

SortedMap<Key,Value> expectedOutput = new TreeMap<>();
input.forEach((k, v) -> {
if (k.getColumnQualifierData().toString().contains("abcdef") || k.isDeleted()) {
expectedOutput.put(k, v);
}
});

assertFalse(expectedOutput.isEmpty());
checkEntries(gi, expectedOutput);
}

@Test
public void testMatchVisibility() throws Exception {
GrepIterator gi = new GrepIterator();
IteratorSetting is = new IteratorSetting(1, GrepIterator.class);

GrepIterator.setTerm(is, "abcdef");

GrepIterator.matchRow(is, false);
GrepIterator.matchColumnFamily(is, false);
GrepIterator.matchColumnQualifier(is, false);
GrepIterator.matchColumnVisibility(is, true);
GrepIterator.matchValue(is, false);

gi.init(new SortedMapIterator(input), is.getOptions(), null);
gi.seek(new Range(), EMPTY_COL_FAMS, false);

SortedMap<Key,Value> expectedOutput = new TreeMap<>();
input.forEach((k, v) -> {
if (k.getColumnVisibilityData().toString().contains("abcdef") || k.isDeleted()) {
expectedOutput.put(k, v);
}
});

assertFalse(expectedOutput.isEmpty());
checkEntries(gi, expectedOutput);
}

@Test
public void testMatchValue() throws Exception {
GrepIterator gi = new GrepIterator();
IteratorSetting is = new IteratorSetting(1, GrepIterator.class);

GrepIterator.setTerm(is, "abcdef");

GrepIterator.matchRow(is, false);
GrepIterator.matchColumnFamily(is, false);
GrepIterator.matchColumnQualifier(is, false);
GrepIterator.matchColumnVisibility(is, false);
GrepIterator.matchValue(is, true);

gi.init(new SortedMapIterator(input), is.getOptions(), null);
gi.seek(new Range(), EMPTY_COL_FAMS, false);

SortedMap<Key,Value> expectedOutput = new TreeMap<>();
input.forEach((k, v) -> {
if (v.toString().contains("abcdef") || k.isDeleted()) {
expectedOutput.put(k, v);
}
});

assertFalse(expectedOutput.isEmpty());
checkEntries(gi, expectedOutput);
}
}

0 comments on commit 1e75ed1

Please sign in to comment.