Skip to content

Commit

Permalink
adding tests to verify that load csv fails when attempting to load fr…
Browse files Browse the repository at this point in the history 
…om a blocked ip range
  • Loading branch information
@phil198
phil198 committed Jan 18, 2022
1 parent a122db1 commit bcc6103
Show file tree
Hide file tree
Showing 4 changed files with 40 additions and 17 deletions.
1 change: 0 additions & 1 deletion core/src/main/java/apoc/load/LoadJson.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ public Stream<MapResult> loadJsonStream(@Name("urlOrKeyOrBinary") Object urlOrKe
headers = null != headers ? headers : new HashMap<>();
headers.putAll(Util.extractCredentialsIfNeeded((String) urlOrKeyOrBinary, failOnError));
}
// TODO FIXME
Stream<Object> stream = JsonUtil.loadJson(urlOrKeyOrBinary,headers,payload, path, failOnError, compressionAlgo, pathOptions, apocConfig);
return stream.flatMap((value) -> {
if (value instanceof Map) {
Expand Down
15 changes: 6 additions & 9 deletions core/src/main/java/apoc/util/JsonUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,17 +97,14 @@ public static Stream<Object> loadJson(Object urlOrBinary, Map<String,Object> hea
try {
if (urlOrBinary instanceof String) {
String url = (String) urlOrBinary;
try
{
URL u = new URL( url );
try {
URL u = new URL(url);
String protocol = u.getProtocol();
if ( protocol.equals( "http" ) || protocol.equals( "https" ) || protocol.equals( "ftp" ) )
{
webAccessRule.validate( apocConfig.getNeo4jConfig(), new URL( url ) );
if (protocol.equals("http") || protocol.equals("https") || protocol.equals("ftp")) {
webAccessRule.validate(apocConfig.getNeo4jConfig(), new URL(url));
}
} catch ( Exception e )
{
throw new RuntimeException( e );
} catch (Exception e) {
throw new RuntimeException(e);
}
urlOrBinary = Util.getLoadUrlByConfigFile("json", url, "url").orElse(url);
}
Expand Down
25 changes: 25 additions & 0 deletions core/src/test/java/apoc/load/LoadJsonTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,17 @@
import apoc.util.TestUtil;
import apoc.util.Util;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.net.util.SubnetUtils;
import org.junit.*;
import org.mockserver.client.server.MockServerClient;
import org.mockserver.integration.ClientAndServer;
import org.mockserver.model.Header;

import org.neo4j.configuration.GraphDatabaseInternalSettings;
import org.neo4j.exceptions.CypherExecutionException;
import org.neo4j.graphdb.QueryExecutionException;
import org.neo4j.graphdb.Result;
import org.neo4j.graphdb.security.URLAccessValidationError;
import org.neo4j.internal.helpers.collection.Iterators;
import org.neo4j.test.rule.DbmsRule;
import org.neo4j.test.rule.ImpermanentDbmsRule;
Expand Down Expand Up @@ -60,12 +65,17 @@ public static void stopServer() {
// .withSetting(ApocSettings.apoc_import_file_enabled, true)
// .withSetting(ApocSettings.apoc_import_file_use__neo4j__config, false);

@Rule
public DbmsRule db2 = new ImpermanentDbmsRule()
.withSetting(GraphDatabaseInternalSettings.cypher_ip_blocklist, List.of(new SubnetUtils("127.168.0.0/8")));

@Before public void setUp() throws Exception {
apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true);
apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false);
apocConfig().setProperty("apoc.json.zip.url", "https://github.com/neo4j-contrib/neo4j-apoc-procedures/blob/3.4/src/test/resources/testload.zip?raw=true!person.json");
apocConfig().setProperty("apoc.json.simpleJson.url", ClassLoader.getSystemResource("map.json").toString());
TestUtil.registerProcedure(db, LoadJson.class);
TestUtil.registerProcedure(db2, LoadJson.class);
}

@Test public void testLoadJson() throws Exception {
Expand All @@ -76,6 +86,21 @@ public static void stopServer() {
});
}

@Test public void testLoadJsonFromBlockedIpRange() throws Exception {
var protocols = List.of("https", "http", "ftp");

for (var protocol: protocols) {
QueryExecutionException e = Assert.assertThrows(QueryExecutionException.class,
() -> testCall(db2,
"CALL apoc.load.json('" + protocol + "://localhost/test.csv')",
map(),
(r) -> {}
)
);
assertTrue(e.getMessage().contains("access to localhost/127.0.0.1 is blocked via the configuration property unsupported.dbms.cypher_ip_blocklist"));
}
}

@Test
public void testLoadMultiJsonWithBinary() {
testResult(db, "CALL apoc.load.jsonParams($url, null, null, null, $config)",
Expand Down
16 changes: 9 additions & 7 deletions full/src/main/java/apoc/es/ElasticSearch.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
@Extended
public class ElasticSearch {

private final static LoadJson loadJson = new LoadJson();

private final static String fullQueryTemplate = "/%s/%s/%s?%s";

// /{index}/{type}/_search?{query}
Expand Down Expand Up @@ -138,33 +140,33 @@ protected String toQueryParams(Object query) {
@Description("apoc.es.stats(host-url-Key) - elastic search statistics")
public Stream<MapResult> stats(@Name("host") String hostOrKey) {
String url = getElasticSearchUrl(hostOrKey);
return (new LoadJson()).loadJsonStream(url + "/_stats", null, null);
return loadJson.loadJsonStream(url + "/_stats", null, null);
}

@Procedure
@Description("apoc.es.get(host-or-port,index-or-null,type-or-null,id-or-null,query-or-null,payload-or-null) yield value - perform a GET operation on elastic search")
public Stream<MapResult> get(@Name("host") String hostOrKey, @Name("index") String index, @Name("type") String type, @Name("id") String id, @Name("query") Object query, @Name("payload") Object payload) {
return (new LoadJson()).loadJsonStream(getQueryUrl(hostOrKey, index, type, id, query), map("content-type",contentType(payload)), toPayload(payload));
return loadJson.loadJsonStream(getQueryUrl(hostOrKey, index, type, id, query), map("content-type",contentType(payload)), toPayload(payload));
}

@Procedure
@Description("apoc.es.query(host-or-port,index-or-null,type-or-null,query-or-null,payload-or-null) yield value - perform a SEARCH operation on elastic search")
public Stream<MapResult> query(@Name("host") String hostOrKey, @Name("index") String index, @Name("type") String type, @Name("query") Object query, @Name("payload") Object payload) {
return (new LoadJson()).loadJsonStream(getSearchQueryUrl(hostOrKey, index, type, query), map("content-type",contentType(payload)), toPayload(payload));
return loadJson.loadJsonStream(getSearchQueryUrl(hostOrKey, index, type, query), map("content-type",contentType(payload)), toPayload(payload));
}

@Procedure
@Description("apoc.es.getRaw(host-or-port,path,payload-or-null) yield value - perform a raw GET operation on elastic search")
public Stream<MapResult> getRaw(@Name("host") String hostOrKey, @Name("path") String suffix, @Name("payload") Object payload) {
String url = getElasticSearchUrl(hostOrKey);
return (new LoadJson()).loadJsonStream(url + "/" + suffix, map("content-type",contentType(payload)), toPayload(payload));
return loadJson.loadJsonStream(url + "/" + suffix, map("content-type",contentType(payload)), toPayload(payload));
}

@Procedure
@Description("apoc.es.postRaw(host-or-port,path,payload-or-null) yield value - perform a raw POST operation on elastic search")
public Stream<MapResult> postRaw(@Name("host") String hostOrKey, @Name("path") String suffix, @Name("payload") Object payload) {
String url = getElasticSearchUrl(hostOrKey);
return (new LoadJson()).loadJsonStream(url + "/" + suffix, map("method", "POST","content-type",contentType(payload)), toPayload(payload));
return loadJson.loadJsonStream(url + "/" + suffix, map("method", "POST","content-type",contentType(payload)), toPayload(payload));
}

@Procedure
Expand All @@ -174,7 +176,7 @@ public Stream<MapResult> post(@Name("host") String hostOrKey, @Name("index") Str
{
payload = Collections.emptyMap();
}
return (new LoadJson()).loadJsonStream(getQueryUrl(hostOrKey, index, type, null, query), map("method", "POST","content-type",contentType(payload)), toPayload(payload));
return loadJson.loadJsonStream(getQueryUrl(hostOrKey, index, type, null, query), map("method", "POST","content-type",contentType(payload)), toPayload(payload));
}

@Procedure
Expand All @@ -184,6 +186,6 @@ public Stream<MapResult> put(@Name("host") String hostOrKey, @Name("index") Stri
{
payload = Collections.emptyMap();
}
return (new LoadJson()).loadJsonStream(getQueryUrl(hostOrKey, index, type, id, query), map("method", "PUT","content-type",contentType(payload)), toPayload(payload));
return loadJson.loadJsonStream(getQueryUrl(hostOrKey, index, type, id, query), map("method", "PUT","content-type",contentType(payload)), toPayload(payload));
}
}

0 comments on commit bcc6103

Please sign in to comment.