Skip to content

Commit

Permalink
fix: remove from first-pass since no benefit
Browse files Browse the repository at this point in the history
  • Loading branch information
gabyx committed Jul 12, 2024
1 parent 2a30234 commit 8c15fb0
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 37 deletions.
2 changes: 1 addition & 1 deletion src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ fn main() {
match cli.command {
Subcommands::Index(args) => {
info!(log, "Args: {:?}", args);
create_type_map(&args.input, &args.output, parallelize)
create_type_map(&args.input, &args.output)
}
Subcommands::Pseudo(args) => {
info!(log, "Args: {:?}", args);
Expand Down
46 changes: 11 additions & 35 deletions src/pass_first.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
use rayon::prelude::*;
use rio_api::parser::TriplesParser;
use rio_turtle::TurtleError;
use std::{io::Write, path::Path, sync::Mutex};
use std::{io::Write, path::Path};

use crate::{
io,
Expand All @@ -21,42 +20,19 @@ fn index_triple(t: Triple, out: &mut impl Write) {
}
}

pub fn create_type_map(input: &Path, output: &Path, parallel: bool) {
pub fn create_type_map(input: &Path, output: &Path) {
let buf_in = io::get_reader(input);
let mut buf_out = io::get_writer(output);
let mut triples = io::parse_ntriples(buf_in);

if parallel {
// Make a parallel triple iterator over `rdf_types::Triple`.
// We have to wrap the `buf_out` with a `Mutex` to make it
// writable by multiple threads.
// NOTE: Weird `rio_api::into_iter` implementation, why does it use a full-blown
// `Vec<T>`, this could be simpler.
//
let buf_out = Mutex::new(buf_out);
let it = triples
.into_iter(|t: TripleView| Result::<Triple, TurtleError>::Ok(t.into()))
.par_bridge();

// Iterate in parallel over the triples.
it.for_each(|r| match r {
Err(e) => panic!("Parsing error occured: {e}"),
Ok(t) => {
let mut guard = buf_out.lock().unwrap();
index_triple(t, guard.by_ref());
}
});
} else {
// Run the loop single-threaded.
while !triples.is_end() {
let _ = triples
.parse_step(&mut |t: TripleView| {
index_triple(t.into(), &mut buf_out);
Result::<(), TurtleError>::Ok(())
})
.inspect_err(|e| {
panic!("Parsing error occured: {e}");
});
}
while !triples.is_end() {
let _ = triples
.parse_step(&mut |t: TripleView| {
index_triple(t.into(), &mut buf_out);
Result::<(), TurtleError>::Ok(())
})
.inspect_err(|e| {
panic!("Parsing error occured: {e}");
});
}
}
2 changes: 1 addition & 1 deletion src/pass_second.rs
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ pub fn pseudonymize_graph(
// We have to wrap the `buf_out` with a `Mutex` to make it
// writable by multiple threads.
// Also: The Mutex<BufWriter> will be double locked 🤔
// if `stdout` is used since the
// if `stdout` is used since its locked internally.
// std::io::stdout() is already locked internall.
//
// NOTE: Weird `rio_api::into_iter` implementation, why does it use a full-blown
Expand Down

0 comments on commit 8c15fb0

Please sign in to comment.