Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

move lock: in-place dependency graph updates #16788

Merged
merged 2 commits into from
Mar 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,6 @@ async fn test_generate_lock_file() {
version = 1
manifest_digest = "1401DE1C3C3FF6D20EB27741A0A7B5D61E34836CB6C90ECC2F2DE97C47B4D0F9"
deps_digest = "3C4103934B1E040BB6B23F1D610B4EF9F2F1166A50A104EADCF77467C004C600"

dependencies = [
{ name = "Examples" },
{ name = "Sui" },
Expand Down
57 changes: 55 additions & 2 deletions external-crates/move/crates/move-package/src/lock_file/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,11 @@ use anyhow::{anyhow, bail, Context, Result};
use serde::{Deserialize, Serialize};
use tempfile::NamedTempFile;
use toml::value::Value;
use toml_edit::{Item::Value as EItem, Value as EValue};
use toml_edit::{
ArrayOfTables,
Item::{self, Value as EItem},
Value as EValue,
};

use move_compiler::editions::{Edition, Flavor};

Expand Down Expand Up @@ -193,7 +197,56 @@ pub(crate) fn write_prologue(
})?;

write!(file, "{}", prologue)?;
Ok(())
}

pub fn update_dependency_graph(
file: &mut LockFile,
manifest_digest: String,
deps_digest: String,
dependencies: Option<toml_edit::Value>,
dev_dependencies: Option<toml_edit::Value>,
packages: Option<ArrayOfTables>,
) -> Result<()> {
Comment on lines +203 to +210
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updates the relevant TOML values for the dependency graph in file without affecting any other part of the contents.

Currently, file passed here is always generated from scratch (as before). But in upcoming changes, file will be an existing lock file, e.g., containing automated address management info, that will now remain intact when the dependency graph is updated via this function.

use toml_edit::value;
let mut toml_string = String::new();
file.read_to_string(&mut toml_string)?;
let mut toml = toml_string.parse::<toml_edit::Document>()?;
let move_table = toml
.entry("move")
.or_insert(Item::Table(toml_edit::Table::new()))
.as_table_mut()
.ok_or_else(|| anyhow!("Could not find or create move table in Move.lock"))?;

// Update `manifest_digest` and `deps_digest` in `[move]` table section.
move_table["manifest_digest"] = value(manifest_digest);
move_table["deps_digest"] = value(deps_digest);

// Update `dependencies = [ ... ]` in `[move]` table section.
if let Some(dependencies) = dependencies {
move_table["dependencies"] = Item::Value(dependencies.clone());
} else {
move_table.remove("dependencies");
}

// Update `dev-dependencies = [ ... ]` in `[move]` table section.
if let Some(dev_dependencies) = dev_dependencies {
move_table["dev-dependencies"] = Item::Value(dev_dependencies.clone());
} else {
move_table.remove("dev-dependencies");
}

// Update the [[move.package]] Array of Tables.
if let Some(packages) = packages {
toml["move"]["package"] = Item::ArrayOfTables(packages.clone());
} else if let Some(packages_table) = toml["move"]["package"].as_table_mut() {
packages_table.remove("package");
}

file.set_len(0)?;
file.rewind()?;
write!(file, "{}", toml)?;
file.flush()?;
Ok(())
}

Expand Down Expand Up @@ -265,7 +318,7 @@ pub fn update_managed_address(
environment: &str,
managed_address_update: ManagedAddressUpdate,
) -> Result<()> {
use toml_edit::{value, Document, Item, Table};
use toml_edit::{value, Document, Table};

let mut toml_string = String::new();
file.read_to_string(&mut toml_string)?;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use std::{
collections::{btree_map::Entry, BTreeMap, BTreeSet, VecDeque},
fmt,
fs::File,
io::{BufWriter, Read, Write},
io::{Read, Write},
path::{Path, PathBuf},
process::Command,
};
Expand Down Expand Up @@ -1214,12 +1214,8 @@ impl DependencyGraph {
/// This operation fails, writing nothing, if the graph contains a cycle, and can fail with an
/// undefined output if it cannot be represented in a TOML file.
pub fn write_to_lock(&self, install_dir: PathBuf) -> Result<LockFile> {
let lock = LockFile::new(
install_dir,
self.manifest_digest.clone(),
self.deps_digest.clone(),
)?;
let mut writer = BufWriter::new(&*lock);
use fmt::Write;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: I would personally just put the trait use statements at the top-level (also applies to Seek below)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For sure! In this case, there is already a io::Write at the top that conflicts. I'm not sure there's an elegant way to overcome that?

let mut writer = String::new();

self.write_dependencies_to_lock(self.root_package_id, &mut writer)?;

Expand All @@ -1235,15 +1231,47 @@ impl DependencyGraph {
self.write_dependencies_to_lock(*id, &mut writer)?;
}

writer.flush()?;
std::mem::drop(writer);
let mut dependencies = None;
let mut dev_dependencies = None;
let mut packages = None;
if !writer.is_empty() {
let toml = writer.parse::<toml_edit::Document>()?;
if let Some(value) = toml.get("dependencies").and_then(|v| v.as_value()) {
dependencies = Some(value.clone());
}
if let Some(value) = toml.get("dev-dependencies").and_then(|v| v.as_value()) {
dev_dependencies = Some(value.clone());
}
packages = toml
.get("move")
.and_then(|m| m.as_table())
.and_then(|move_table| move_table.get("package"))
.and_then(|v| v.as_array_of_tables().cloned());
}

use std::io::Seek;
let mut lock = LockFile::new(
install_dir,
self.manifest_digest.clone(),
self.deps_digest.clone(),
)?;
lock.flush()?;
lock.rewind()?;
Comment on lines +1252 to +1259
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We still unconditionally create a Move.lock from scratch as before. But this time, we will update that Move.lock with the update_dependency_graph function and edit in-place (right after this comment).

We rewind the lock file cursor so that it's ready to be read by the update_dependency_graph function. This is how all the update_* functions are set up to work in schema.rs (i.e., they expect a file handle they can read a lock string from).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does it make sense to just work with strings (or some other in-memory representation) everywhere instead of the file that you need to keep rewinding, then at the end, you can put it back into the file?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I'd like something like that, but it's tricky. In effect we'd need to thread that string along wherever we'd like to avoid a write to the file, which is intimidating--mostly because the control flow is tricky, and the points at which various things care to commit or write to the file is spread out over that control flow (meaning the callers will then read/write anyway).

I'm happy to try rework this later, but consistent updates to the lock file path this way abstracts away from the callers and avoids passing a string around. Because one thing I think doesn't work is to have, e.g., dependency_graph or any other component work with strings be tempted to decide "ok since I have a string I care about, I'm writing to the lock file right here right now!". If we can have updates go through a predictable interface (lockfile schema update* functions) then we can find/reason about stateful effects to the lock (with find-references).

Again I may revisit and have these return strings, it might make sense, though there's also a chance that change will clutter up callers with needing to write to disk at various places.


schema::update_dependency_graph(
&mut lock,
self.manifest_digest.clone(),
self.deps_digest.clone(),
dependencies,
dev_dependencies,
packages,
)?;
Ok(lock)
}

/// Helper function to output the dependencies and dev-dependencies of `name` from this
/// dependency graph, to the lock file under `writer`.
fn write_dependencies_to_lock<W: Write>(
fn write_dependencies_to_lock<W: fmt::Write>(
&self,
id: PackageIdentifier,
writer: &mut W,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,10 @@
version = 1
manifest_digest = "98BBCE8D1C29472825E598691218A0CBC5BDA1A56C4429F5C2311C245DEC28CE"
deps_digest = "060AD7E57DFB13104F21BE5F5C3759D03F0553FC3229247D9A7A6B45F50D03A3"

dependencies = [
{ name = "A" },
{ name = "C" },
]

dev-dependencies = [
{ name = "B" },
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "A4DB860CC2BC78C04706A7383CA52121876F87057538DF814C66F7EE6025E644"
deps_digest = "F8BBB0CCB2491CA29A3DF03D6F92277A4F3574266507ACD77214D37ECA3F3082"

dependencies = [
{ name = "OtherDep", digest = "A666AE1AFDBA7E287476E851C0011503D19EE73E89CBDB307D4E0067E8BBA22C", addr_subst = { "A" = "B" } },
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,9 @@
version = 1
manifest_digest = "CB01A8B6F9859E70A0A5DA10F8547C13EDDD63E9EDF72E930DD37C8EFC41F3F3"
deps_digest = "060AD7E57DFB13104F21BE5F5C3759D03F0553FC3229247D9A7A6B45F50D03A3"

dependencies = [
{ name = "C" },
]

dev-dependencies = [
{ name = "A" },
{ name = "B" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "B06FCED8E0EF6B62EC0B572DC233C0D70206B1C10EEDEA0403CA64AFBB3E439B"
deps_digest = "3C4103934B1E040BB6B23F1D610B4EF9F2F1166A50A104EADCF77467C004C600"

dependencies = [
{ name = "A" },
{ name = "B" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "82E01E336DD3374BDC42CA355AB6ACA69E44DB65E36DC0A831F36B06F6832574"
deps_digest = "060AD7E57DFB13104F21BE5F5C3759D03F0553FC3229247D9A7A6B45F50D03A3"

dependencies = [
{ name = "A" },
{ name = "ADep" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "E1BD09BE802FCCF437672321DEBFD0C8000F9D4A0AC5E54ED432087ABACA9667"
deps_digest = "060AD7E57DFB13104F21BE5F5C3759D03F0553FC3229247D9A7A6B45F50D03A3"

dependencies = [
{ name = "A" },
{ name = "B" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "86DBE490660052E70AF19AEB6DB3CDFE90F770D67E008E12C753575AB346B43C"
deps_digest = "060AD7E57DFB13104F21BE5F5C3759D03F0553FC3229247D9A7A6B45F50D03A3"

dependencies = [
{ name = "A" },
{ name = "B" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "F64B7E3BA42923C4AFD7490B03D34813BA434044DC83D46F869DD5BB9A6052B2"
deps_digest = "060AD7E57DFB13104F21BE5F5C3759D03F0553FC3229247D9A7A6B45F50D03A3"

dependencies = [
{ name = "A" },
{ name = "B" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "B2F5516D0E0D7FC1D1A91EF42181BC28609979CA8D75F7EF01B473AF22303C1D"
deps_digest = "060AD7E57DFB13104F21BE5F5C3759D03F0553FC3229247D9A7A6B45F50D03A3"

dependencies = [
{ name = "A" },
{ name = "B" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "88451CA3B87F330C2224714E02829B02787D5AEA4F9CCD9FF239ED0344CF0632"
deps_digest = "060AD7E57DFB13104F21BE5F5C3759D03F0553FC3229247D9A7A6B45F50D03A3"

dependencies = [
{ name = "A" },
{ name = "B" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "2CE4CA7B1785FEAE60C59A993DB1182E09DB665E694C6104DF566E065752C030"
deps_digest = "3C4103934B1E040BB6B23F1D610B4EF9F2F1166A50A104EADCF77467C004C600"

dependencies = [
{ name = "A", addr_subst = { "AA" = "0000000000000000000000000000000000000000000000000000000000000001" } },
{ name = "B", addr_subst = { "BA" = "0000000000000000000000000000000000000000000000000000000000000001" } },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "373B3598A66D05FBD7DE398D2F588C9C79C9F38757140E8717C1D851CBC5C15F"
deps_digest = "3C4103934B1E040BB6B23F1D610B4EF9F2F1166A50A104EADCF77467C004C600"

dependencies = [
{ name = "A" },
{ name = "B" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "8D09D19521F36950C0698F14ED09FE4F6175C022796F1E401F2F5A1BCA6FCE98"
deps_digest = "F8BBB0CCB2491CA29A3DF03D6F92277A4F3574266507ACD77214D37ECA3F3082"

dependencies = [
{ name = "A" },
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "725168ABE1A1677C13C020BCA80D7C059B53A9C7E2E4D15FD0BE671E0D2A87B0"
deps_digest = "F8BBB0CCB2491CA29A3DF03D6F92277A4F3574266507ACD77214D37ECA3F3082"

dependencies = [
{ name = "A" },
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,9 @@
version = 1
manifest_digest = "84A0B503BE9F9B341AC66860713D12704876E5C2425891E3B94626B12E4313E5"
deps_digest = "3C4103934B1E040BB6B23F1D610B4EF9F2F1166A50A104EADCF77467C004C600"

dependencies = [
{ name = "A" },
]

dev-dependencies = [
{ name = "B" },
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "2401A97D2FE979752E0726963BFC3F677A4FA8041F3C6FAE5A4302401B165463"
deps_digest = "3C4103934B1E040BB6B23F1D610B4EF9F2F1166A50A104EADCF77467C004C600"

dependencies = [
{ name = "A" },
{ name = "ADep" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "C5119AF40DAACF1DF25E8E9DE830EC0EA839A29E62D7081BF217B49B10BD6980"
deps_digest = "F8BBB0CCB2491CA29A3DF03D6F92277A4F3574266507ACD77214D37ECA3F3082"

dependencies = [
{ name = "A" },
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "846084292F5A72B3CFE28402E76C1F3F172C3001C90FAAC19221D4B178A84EDB"
deps_digest = "3C4103934B1E040BB6B23F1D610B4EF9F2F1166A50A104EADCF77467C004C600"

dependencies = [
{ name = "C" },
{ name = "D" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "598DCC919F7378E59F328E1B448D1AAC70B8F34894146860B3ABF46600F9F79B"
deps_digest = "F8BBB0CCB2491CA29A3DF03D6F92277A4F3574266507ACD77214D37ECA3F3082"

dependencies = [
{ name = "MoveNursery" },
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "CD8925BBAAE2A64C347B3D48466097B55E04876C226546516D83EDC58C0BBB3A"
deps_digest = "F8BBB0CCB2491CA29A3DF03D6F92277A4F3574266507ACD77214D37ECA3F3082"

dependencies = [
{ name = "Nested" },
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "CD8925BBAAE2A64C347B3D48466097B55E04876C226546516D83EDC58C0BBB3A"
deps_digest = "F8BBB0CCB2491CA29A3DF03D6F92277A4F3574266507ACD77214D37ECA3F3082"

dependencies = [
{ name = "Nested" },
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "5005BF461ECC5281FE314371B38D8C163C5395D7455B91BEA0F63C1BCFD57551"
deps_digest = "060AD7E57DFB13104F21BE5F5C3759D03F0553FC3229247D9A7A6B45F50D03A3"

dependencies = [
{ name = "More" },
{ name = "Nested" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "0C66C2C067539518C3189E86B5A09D478C872C0F97ACE6D707AE9753319E56AA"
deps_digest = "F8BBB0CCB2491CA29A3DF03D6F92277A4F3574266507ACD77214D37ECA3F3082"

dependencies = [
{ name = "OtherDep", addr_subst = { "A" = "B" } },
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "9DE3FDE63E3A9E1827A3D28769FE454A274BEF7DC0514AEEA9076DE876F2C361"
deps_digest = "F8BBB0CCB2491CA29A3DF03D6F92277A4F3574266507ACD77214D37ECA3F3082"

dependencies = [
{ name = "OtherDep", digest = "BAD_DIGEST", addr_subst = { "A" = "B" } },
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "340302CAC58F9844D483E9F19F08D9578B90043E621D953A62A4B679F8D98896"
deps_digest = "F8BBB0CCB2491CA29A3DF03D6F92277A4F3574266507ACD77214D37ECA3F3082"

dependencies = [
{ name = "OtherDep", addr_subst = { "A" = "B" } },
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "6F18190939664D7ECC8F2DC327E079037A743E0CFF3FA5F72DABD2B6B5C3D200"
deps_digest = "3C4103934B1E040BB6B23F1D610B4EF9F2F1166A50A104EADCF77467C004C600"

dependencies = [
{ name = "A-resolved", addr_subst = { "AA" = "0000000000000000000000000000000000000000000000000000000000000001" } },
{ name = "B-resolved", addr_subst = { "BA" = "0000000000000000000000000000000000000000000000000000000000000001" } },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "A152DBB11C386226B7A6435D66090103E0CC19330A38251784E4D2D0C0EF57A5"
deps_digest = "3C4103934B1E040BB6B23F1D610B4EF9F2F1166A50A104EADCF77467C004C600"

dependencies = [
{ name = "A" },
{ name = "B" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
version = 1
manifest_digest = "88C81CA573217383E355274A37F0C6171DDE43ED835A166B0D6446C294788F65"
deps_digest = "3C4103934B1E040BB6B23F1D610B4EF9F2F1166A50A104EADCF77467C004C600"

dependencies = [
{ name = "A" },
{ name = "B" },
Expand Down
Loading
Loading