From 605314aa64d9d02ccf90b28cf5016b8fdd468369 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Wed, 27 Oct 2021 16:15:28 -0500 Subject: [PATCH 01/22] add first draft of proposal --- PLUGIN-DESIGN.md | 265 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 PLUGIN-DESIGN.md diff --git a/PLUGIN-DESIGN.md b/PLUGIN-DESIGN.md new file mode 100644 index 000000000..a0c039055 --- /dev/null +++ b/PLUGIN-DESIGN.md @@ -0,0 +1,265 @@ +# User Plugins Design Proposal + +## Background + +This is a design proposal for the long-standing [#55](https://github.com/rojo-rbx/rojo/issues/55) +desire to add user plugins to Rojo for things like source file transformation and instance tree +transformation. + +As discussed in [#55](https://github.com/rojo-rbx/rojo/issues/55) and as initially explored in +[#257](https://github.com/rojo-rbx/rojo/pull/257), plugins as Lua scripts seem to be a good starting +point. This is quite similar to the way Rollup.js plugins work, although they are implemented with +JS. Rollup.js is a bundler which actually performs a similar job to Rojo in the JS world by taking a +number of source files and converting them into a single output bundle. + +This proposal takes strong influence from the [Rollup.js plugins +API](https://rollupjs.org/guide/en/#plugins-overview) which is a joy to use for both plugin +developers and end-users. + +## Project file changes + +Add a new top-level field to the project file format: + +- `plugins`: An array of [Plugin Descriptions](#plugin-description). + - **Optional** + - Default is `[]` + +### Plugin description + +Either a `String` value or an object with the fields: + +- `source`: A filepath or URL to the Lua source file of the plugin. + - **Required** +- `options`: Any JSON dictionary. The options that will be passed to the plugin. + - **Optional** + - Default is `{}` + +In the case that the value is just a `String`, it is interpreted as an object with the `source` +field set to its value, and `options` set to its default. + +### Example + +```json +{ + "name": "ProjectWithPlugins", + "tree": { + "$className": "DataModel", + "ServerScriptService": { + "$className": "ServerScriptService", + "$path": "src" + } + }, + "plugins": [ + "local-plugin.lua", + "github.com/owner/remote-plugin-from-tag@v1.0.0", + "github.com/owner/remote-plugin-from-head", + { "source": "plugin-with-options.lua", "options": { "some": "option" } } + ] +} +``` + +## Plugin scripts + +Plugin scripts should return a `CreatePluginFunction`: + +```luau +-- Types provided in luau format + +type PluginInstance = { + projectDescription?: (project: ProjectDescription) -> (), + syncStart?: () -> (), + syncEnd?: () -> (), + resolve?: (id: string) -> string, + middleware?: (id: string) -> string, + load?: (id: string) -> string, +} + +-- TODO: Define properly. For now, this is basically just the JSON converted to Lua +type ProjectDescription = { ... } + +type CreatePluginFunction = (options: {[string]: any}) -> PluginInstance +``` + +In this way, plugins have the opportunity to customize their hooks based on the options provided by +the user in the project file. + +## Plugin environment + +The plugin environment is created in the following way: + +1. Create a new Lua context. +1. Initialize an empty `_G.plugins` table. +1. For each plugin description in the project file: + 1. Convert the plugin options from the project file from JSON to a Lua table. + 1. If the `source` field is a GitHub URL, download the plugin directory from the repo with the + specified version tag (if no tag, from the head of the default branch) into a local + `.rojo-plugins` directory with the repo identifier as its name. It is recommended that users + add `.rojo-plugins` to their `.gitignore` file. The root of the plugin will be called + `main.lua`. + 1. Load and evaluate the file contents into the Lua context to get a handle to the + `CreatePluginFunction` + 1. Call the `CreatePluginFunction` with the converted options to get a handle of the result. + 1. Push the result at the end of the `_G.plugins` table + +If at any point there is an error in the above steps, Rojo should quit with an appropriate error +message. + +## Plugin hooks + +- `projectDescription(project: ProjectDescription) -> ()` + - Called with a Lua representation of the current project description whenever it has changed. +- `syncStart() -> ()` + - A sync has started. +- `syncEnd() -> ()` + - A sync has finished. +- `resolve(id: string) -> string` + - Takes a file path and returns a new file path that the file should be loaded from instead. + The first plugin to return a non-nil value per id wins. +- `middleware(id: string) -> string` + - Takes a file path and returns a snapshot middleware enum to determine how Rojo should build + the instance tree for the file. The first plugin to return a non-nil value per id wins. +- `load(id: string) -> string` + - Takes a file path and returns the file contents that should be interpreted by Rojo. The + first plugin to return a non-nil value per id wins. + +## Use case analyses + +To demonstrate the effectiveness of this API, pseudo-implementations for a variety of use-cases are +shown using the API. + +### MoonScript transformation + +```lua +local parse = require 'moonscript.parse' +local compile = require 'moonscript.compile' + +return function(options) + return { + load = function(id) + if id:match('%.lua$') then + local file = io.open(id, 'r') + local source = file:read('a') + file:close() + + local tree, err = parse.string(source) + assert(tree, err) + + local lua, err, pos = compile.tree(tree) + if not lua then error(compile.format_error(err, pos, source)) end + + return lua + end + end + } +end +``` + +### Obfuscation/minifier transformation + +```lua +local minifier = require 'minifier.lua' + +return function(options) + return { + load = function(id) + if id:match('%.lua$') then + local file = io.open(id, 'r') + local source = file:read('a') + file:close() + return minifier(source) + end + end + } +end +``` + +### Custom file types + +```lua +return function(options) + return { + middleware = function(id) + if id:match('%.md$') then + return 'json_model' + end + end, + load = function(id) + if id:match('%.md$') then + local file = io.open(id, 'r') + local source = file:read('a') + file:close() + return ('{"ClassName": "StringValue", "Properties": { "Value": "%s" }}'):format(source) + + -- If we converted Lua tables to JSON we could use this nicer syntax: + --[[return { + ClassName = 'StringValue', + Properties = { + Value = source + } + }]] + end + end + } +end +``` + +### Go-style dependency management + +```lua +-- download/caching implementation inline here + +return function(options) + return { + resolve = function(id) + if id:match('^https?://.*%.lua$') then + local cachedId = fromCache(id) + return cachedId or nil + end + end, + load = function(id) + if id:match('^https?://.*%.lua$') then + local cachedId = downloadAndCache(id) + local file = io.open(cachedId, 'r') + local source = file:read('a') + file:close() + return source + end + end + } +end +``` + +### File system requires + +```lua +-- lua parsing/writing implementation here + +return function(options) + local project = nil + + return { + projectDescription = function(newProject) + project = newProject + end, + load = function(id) + if id:match('%.lua$') then + local file = io.open(id, 'r') + local source = file:read('a') + file:close() + + -- This function will look for require 'file/path' statements in the source and replace + -- them with Roblox require(instance.path) statements based on the project's configuration + -- (where certain file paths are mounted) + return replaceRequires(source, project) + end + end + } +end +``` + +## Concerns + +- Some operations will be common in plugins and a set of standardized functions may be helpful, + for example reading files and checking file extensions. This could be provided as a global + library injected in the initialization stage of the Lua context (e.g. + `rojo.fileExtensionMatches(id, ext)`, `rojo.loadFile(id)`, `rojo.toJson(value)`). From 10e980c42de475717c4c75670db62e0ea2f5e406 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Wed, 27 Oct 2021 16:17:56 -0500 Subject: [PATCH 02/22] reordered example --- PLUGIN-DESIGN.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/PLUGIN-DESIGN.md b/PLUGIN-DESIGN.md index a0c039055..0207e9d4b 100644 --- a/PLUGIN-DESIGN.md +++ b/PLUGIN-DESIGN.md @@ -42,19 +42,19 @@ field set to its value, and `options` set to its default. ```json { "name": "ProjectWithPlugins", + "plugins": [ + "local-plugin.lua", + "github.com/owner/remote-plugin-from-tag@v1.0.0", + "github.com/owner/remote-plugin-from-head", + { "source": "plugin-with-options.lua", "options": { "some": "option" } } + ], "tree": { "$className": "DataModel", "ServerScriptService": { "$className": "ServerScriptService", "$path": "src" } - }, - "plugins": [ - "local-plugin.lua", - "github.com/owner/remote-plugin-from-tag@v1.0.0", - "github.com/owner/remote-plugin-from-head", - { "source": "plugin-with-options.lua", "options": { "some": "option" } } - ] + } } ``` From 1a66313661ddadb635d4bf230d49e0ccb6997d79 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Wed, 27 Oct 2021 16:19:23 -0500 Subject: [PATCH 03/22] clarified source field --- PLUGIN-DESIGN.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/PLUGIN-DESIGN.md b/PLUGIN-DESIGN.md index 0207e9d4b..5ac69f4d2 100644 --- a/PLUGIN-DESIGN.md +++ b/PLUGIN-DESIGN.md @@ -28,7 +28,8 @@ Add a new top-level field to the project file format: Either a `String` value or an object with the fields: -- `source`: A filepath or URL to the Lua source file of the plugin. +- `source`: A filepath to the Lua source file of the plugin or a URL to a GitHub repo, optionally + followed by an `@` character and a Git tag. - **Required** - `options`: Any JSON dictionary. The options that will be passed to the plugin. - **Optional** From 5de7fc5b8c5d0f19265c12089928e5f16c09dfb4 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 08:54:56 -0500 Subject: [PATCH 04/22] add todo --- PLUGIN-DESIGN.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/PLUGIN-DESIGN.md b/PLUGIN-DESIGN.md index 5ac69f4d2..b400c4886 100644 --- a/PLUGIN-DESIGN.md +++ b/PLUGIN-DESIGN.md @@ -191,13 +191,13 @@ return function(options) file:close() return ('{"ClassName": "StringValue", "Properties": { "Value": "%s" }}'):format(source) - -- If we converted Lua tables to JSON we could use this nicer syntax: - --[[return { + -- If we had a library of common functions we could use this nicer syntax: + --[[return rojo.toJson({ ClassName = 'StringValue', Properties = { Value = source } - }]] + })]] end end } @@ -258,8 +258,17 @@ return function(options) end ``` +## Implementation priority + +1. Loading plugins from local paths +2. Calling hooks at the appropriate time +3. Loading plugins from remote repos +4. Rojo plugin library + ## Concerns +TODO: Implement a proposal for a rojo plugin library + - Some operations will be common in plugins and a set of standardized functions may be helpful, for example reading files and checking file extensions. This could be provided as a global library injected in the initialization stage of the Lua context (e.g. From 469c29478df3a657453fd0402599e533fee97545 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 09:05:15 -0500 Subject: [PATCH 05/22] added name property to plugin descriptions --- PLUGIN-DESIGN.md | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/PLUGIN-DESIGN.md b/PLUGIN-DESIGN.md index b400c4886..291ff9f60 100644 --- a/PLUGIN-DESIGN.md +++ b/PLUGIN-DESIGN.md @@ -67,6 +67,7 @@ Plugin scripts should return a `CreatePluginFunction`: -- Types provided in luau format type PluginInstance = { + name: string, projectDescription?: (project: ProjectDescription) -> (), syncStart?: () -> (), syncEnd?: () -> (), @@ -105,23 +106,27 @@ The plugin environment is created in the following way: If at any point there is an error in the above steps, Rojo should quit with an appropriate error message. -## Plugin hooks +## Plugin instance +- `name` + - **Required**: The name of the plugin that will be used in error messages, etc. - `projectDescription(project: ProjectDescription) -> ()` - - Called with a Lua representation of the current project description whenever it has changed. + - **Optional**: Called with a Lua representation of the current project description whenever + it has changed. - `syncStart() -> ()` - - A sync has started. + - **Optional**: A sync has started. - `syncEnd() -> ()` - - A sync has finished. + - **Optional**: A sync has finished. - `resolve(id: string) -> string` - - Takes a file path and returns a new file path that the file should be loaded from instead. - The first plugin to return a non-nil value per id wins. + - **Optional**: Takes a file path and returns a new file path that the file should be loaded + from instead. The first plugin to return a non-nil value per id wins. - `middleware(id: string) -> string` - - Takes a file path and returns a snapshot middleware enum to determine how Rojo should build - the instance tree for the file. The first plugin to return a non-nil value per id wins. + - **Optional**: Takes a file path and returns a snapshot middleware enum to determine how Rojo + should build the instance tree for the file. The first plugin to return a non-nil value per + id wins. - `load(id: string) -> string` - - Takes a file path and returns the file contents that should be interpreted by Rojo. The - first plugin to return a non-nil value per id wins. + - **Optional**: Takes a file path and returns the file contents that should be interpreted by + Rojo. The first plugin to return a non-nil value per id wins. ## Use case analyses @@ -136,6 +141,7 @@ local compile = require 'moonscript.compile' return function(options) return { + name = "moonscript", load = function(id) if id:match('%.lua$') then local file = io.open(id, 'r') @@ -162,6 +168,7 @@ local minifier = require 'minifier.lua' return function(options) return { + name = "minifier", load = function(id) if id:match('%.lua$') then local file = io.open(id, 'r') @@ -179,6 +186,7 @@ end ```lua return function(options) return { + name = "markdown-to-stringvalue", middleware = function(id) if id:match('%.md$') then return 'json_model' @@ -204,13 +212,15 @@ return function(options) end ``` -### Go-style dependency management +### Remote file requires ```lua -- download/caching implementation inline here +-- this one is not really working even from a pseudo-implementation perspective return function(options) return { + name = "remote-require", resolve = function(id) if id:match('^https?://.*%.lua$') then local cachedId = fromCache(id) @@ -239,6 +249,7 @@ return function(options) local project = nil return { + name = "require-files", projectDescription = function(newProject) project = newProject end, From 29769606d3db353e95d909014b74c28c34406d7f Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 09:31:44 -0500 Subject: [PATCH 06/22] added basic plugin loading --- src/lib.rs | 1 + src/plugin_env.rs | 86 +++++ src/project.rs | 4 + src/serve_session.rs | 10 + test-projects/plugins/default.project.json | 14 +- test-projects/plugins/load-as-stringvalue.lua | 318 ++++++++++++++++++ test-projects/plugins/src/hello.md | 3 + test-projects/plugins/src/hello.moon | 1 - test-projects/plugins/test-plugin.lua | 20 -- 9 files changed, 430 insertions(+), 27 deletions(-) create mode 100644 src/plugin_env.rs create mode 100644 test-projects/plugins/load-as-stringvalue.lua create mode 100644 test-projects/plugins/src/hello.md delete mode 100644 test-projects/plugins/src/hello.moon delete mode 100644 test-projects/plugins/test-plugin.lua diff --git a/src/lib.rs b/src/lib.rs index 51195b6c1..92b719ac6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,7 @@ mod lua_ast; mod message_queue; mod multimap; mod path_serializer; +mod plugin_env; mod project; mod resolution; mod serve_session; diff --git a/src/plugin_env.rs b/src/plugin_env.rs new file mode 100644 index 000000000..b8dd016aa --- /dev/null +++ b/src/plugin_env.rs @@ -0,0 +1,86 @@ +use rlua::{Function, Lua, Table}; +use std::fs; + +pub struct PluginEnv { + lua: Lua, +} + +impl PluginEnv { + pub fn new() -> Self { + let lua = Lua::new(); + PluginEnv { lua } + } + + pub fn init(&self) -> Result<(), rlua::Error> { + self.lua.context(|lua_ctx| { + let globals = lua_ctx.globals(); + + let plugins_table = lua_ctx.create_table()?; + globals.set("plugins", plugins_table)?; + + let run_plugins_fn = lua_ctx.create_function(|lua_ctx, id: String| { + let plugins: Table = lua_ctx.globals().get("plugins")?; + let id_ref: &str = &id; + for plugin in plugins.sequence_values::() { + let load: Function = plugin?.get("load")?; + load.call(id_ref)?; + } + + Ok(()) + })?; + globals.set("runPlugins", run_plugins_fn)?; + + Ok::<(), rlua::Error>(()) + }) + } + + fn load_plugin_source(&self, plugin_source: &str) -> String { + // TODO: Support downloading and caching plugins + fs::read_to_string(plugin_source).unwrap() + } + + pub fn load_plugin( + &self, + plugin_source: &str, + plugin_options: String, + ) -> Result<(), rlua::Error> { + let plugin_lua = &(self.load_plugin_source(plugin_source)); + + self.lua.context(|lua_ctx| { + let globals = lua_ctx.globals(); + + let create_plugin: Function = lua_ctx.load(plugin_lua).eval()?; + + let plugin_options_table: Table = lua_ctx.load(&plugin_options).eval()?; + let plugin_instance: Table = create_plugin.call(plugin_options_table)?; + + let plugins_table: Table = globals.get("plugins")?; + plugins_table.set(plugins_table.len()? + 1, plugin_instance)?; + + let run_plugins = lua_ctx.create_function(|lua_ctx, id: String| { + let plugins: Table = lua_ctx.globals().get("plugins")?; + let id_ref: &str = &id; + for plugin in plugins.sequence_values::
() { + let load: Function = plugin?.get("load")?; + load.call(id_ref)?; + } + + Ok(()) + })?; + globals.set("runPlugins", run_plugins)?; + + Ok::<(), rlua::Error>(()) + }) + } + + pub fn run_plugins(&self, id: String) -> Result<(), rlua::Error> { + self.lua.context(|lua_ctx| { + let globals = lua_ctx.globals(); + + let run_plugins: Function = globals.get("runPlugins")?; + run_plugins.call(id)?; + + Ok::<(), rlua::Error>(()) + }) + } +} diff --git a/src/project.rs b/src/project.rs index 088e6d976..531abaa56 100644 --- a/src/project.rs +++ b/src/project.rs @@ -40,6 +40,10 @@ pub struct Project { /// The name of the top-level instance described by the project. pub name: String, + // TODO: Support objects as well as strings for plugins (containing 'source' and 'options' properties) + #[serde(default = "Vec::new", skip_serializing_if = "Vec::is_empty")] + pub plugins: Vec, + /// The tree of instances described by this project. Projects always /// describe at least one instance. pub tree: ProjectNode, diff --git a/src/serve_session.rs b/src/serve_session.rs index 13b8037cc..cd3c2fdbc 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -15,6 +15,7 @@ use thiserror::Error; use crate::{ change_processor::ChangeProcessor, message_queue::MessageQueue, + plugin_env::PluginEnv, project::{Project, ProjectError}, session_id::SessionId, snapshot::{ @@ -119,6 +120,15 @@ impl ServeSession { } }; + let plugin_env = PluginEnv::new(); + plugin_env.init().unwrap(); + + for plugin in root_project.plugins.iter() { + plugin_env + .load_plugin(plugin, "{extensions = {'.md'}}".to_string()) + .unwrap(); + } + let mut tree = RojoTree::new(InstanceSnapshot::new()); let root_id = tree.get_root_id(); diff --git a/test-projects/plugins/default.project.json b/test-projects/plugins/default.project.json index 02387e7e3..fb49e4fed 100644 --- a/test-projects/plugins/default.project.json +++ b/test-projects/plugins/default.project.json @@ -1,9 +1,11 @@ { "name": "plugins", + "plugins": ["load-as-stringvalue.lua"], "tree": { - "$path": "src" - }, - "plugins": [ - "test-plugin.lua" - ] -} \ No newline at end of file + "$className": "DataModel", + "ReplicatedStorage": { + "$className": "ReplicatedStorage", + "$path": "src" + } + } +} diff --git a/test-projects/plugins/load-as-stringvalue.lua b/test-projects/plugins/load-as-stringvalue.lua new file mode 100644 index 000000000..f9707ae27 --- /dev/null +++ b/test-projects/plugins/load-as-stringvalue.lua @@ -0,0 +1,318 @@ +print('[plugin] loading: load-as-stringvalue.lua') + +local function loadRepr() + local defaultSettings = { + pretty = false; + robloxFullName = false; + robloxProperFullName = true; + robloxClassName = true; + tabs = false; + semicolons = false; + spaces = 3; + sortKeys = true; + } + + -- lua keywords + local keywords = {["and"]=true, ["break"]=true, ["do"]=true, ["else"]=true, + ["elseif"]=true, ["end"]=true, ["false"]=true, ["for"]=true, ["function"]=true, + ["if"]=true, ["in"]=true, ["local"]=true, ["nil"]=true, ["not"]=true, ["or"]=true, + ["repeat"]=true, ["return"]=true, ["then"]=true, ["true"]=true, ["until"]=true, ["while"]=true} + + local function isLuaIdentifier(str) + if type(str) ~= "string" then return false end + -- must be nonempty + if str:len() == 0 then return false end + -- can only contain a-z, A-Z, 0-9 and underscore + if str:find("[^%d%a_]") then return false end + -- cannot begin with digit + if tonumber(str:sub(1, 1)) then return false end + -- cannot be keyword + if keywords[str] then return false end + return true + end + + -- works like Instance:GetFullName(), but invalid Lua identifiers are fixed (e.g. workspace["The Dude"].Humanoid) + local function properFullName(object, usePeriod) + if object == nil or object == game then return "" end + + local s = object.Name + local usePeriod = true + if not isLuaIdentifier(s) then + s = ("[%q]"):format(s) + usePeriod = false + end + + if not object.Parent or object.Parent == game then + return s + else + return properFullName(object.Parent) .. (usePeriod and "." or "") .. s + end + end + + local depth = 0 + local shown + local INDENT + local reprSettings + + local function repr(value, reprSettings) + reprSettings = reprSettings or defaultSettings + INDENT = (" "):rep(reprSettings.spaces or defaultSettings.spaces) + if reprSettings.tabs then + INDENT = "\t" + end + + local v = value --args[1] + local tabs = INDENT:rep(depth) + + if depth == 0 then + shown = {} + end + if type(v) == "string" then + return ("%q"):format(v) + elseif type(v) == "number" then + if v == math.huge then return "math.huge" end + if v == -math.huge then return "-math.huge" end + return tonumber(v) + elseif type(v) == "boolean" then + return tostring(v) + elseif type(v) == "nil" then + return "nil" + elseif type(v) == "table" and type(v.__tostring) == "function" then + return tostring(v.__tostring(v)) + elseif type(v) == "table" and getmetatable(v) and type(getmetatable(v).__tostring) == "function" then + return tostring(getmetatable(v).__tostring(v)) + elseif type(v) == "table" then + if shown[v] then return "{CYCLIC}" end + shown[v] = true + local str = "{" .. (reprSettings.pretty and ("\n" .. INDENT .. tabs) or "") + local isArray = true + for k, v in pairs(v) do + if type(k) ~= "number" then + isArray = false + break + end + end + if isArray then + for i = 1, #v do + if i ~= 1 then + str = str .. (reprSettings.semicolons and ";" or ",") .. (reprSettings.pretty and ("\n" .. INDENT .. tabs) or " ") + end + depth = depth + 1 + str = str .. repr(v[i], reprSettings) + depth = depth - 1 + end + else + local keyOrder = {} + local keyValueStrings = {} + for k, v in pairs(v) do + depth = depth + 1 + local kStr = isLuaIdentifier(k) and k or ("[" .. repr(k, reprSettings) .. "]") + local vStr = repr(v, reprSettings) + --[[str = str .. ("%s = %s"):format( + isLuaIdentifier(k) and k or ("[" .. repr(k, reprSettings) .. "]"), + repr(v, reprSettings) + )]] + table.insert(keyOrder, kStr) + keyValueStrings[kStr] = vStr + depth = depth - 1 + end + if reprSettings.sortKeys then table.sort(keyOrder) end + local first = true + for _, kStr in pairs(keyOrder) do + if not first then + str = str .. (reprSettings.semicolons and ";" or ",") .. (reprSettings.pretty and ("\n" .. INDENT .. tabs) or " ") + end + str = str .. ("%s = %s"):format(kStr, keyValueStrings[kStr]) + first = false + end + end + shown[v] = false + if reprSettings.pretty then + str = str .. "\n" .. tabs + end + str = str .. "}" + return str + elseif typeof then + -- Check Roblox types + if typeof(v) == "Instance" then + return (reprSettings.robloxFullName + and (reprSettings.robloxProperFullName and properFullName(v) or v:GetFullName()) + or v.Name) .. (reprSettings.robloxClassName and ((" (%s)"):format(v.ClassName)) or "") + elseif typeof(v) == "Axes" then + local s = {} + if v.X then table.insert(s, repr(Enum.Axis.X, reprSettings)) end + if v.Y then table.insert(s, repr(Enum.Axis.Y, reprSettings)) end + if v.Z then table.insert(s, repr(Enum.Axis.Z, reprSettings)) end + return ("Axes.new(%s)"):format(table.concat(s, ", ")) + elseif typeof(v) == "BrickColor" then + return ("BrickColor.new(%q)"):format(v.Name) + elseif typeof(v) == "CFrame" then + return ("CFrame.new(%s)"):format(table.concat({v:GetComponents()}, ", ")) + elseif typeof(v) == "Color3" then + return ("Color3.new(%d, %d, %d)"):format(v.r, v.g, v.b) + elseif typeof(v) == "ColorSequence" then + if #v.Keypoints > 2 then + return ("ColorSequence.new(%s)"):format(repr(v.Keypoints, reprSettings)) + else + if v.Keypoints[1].Value == v.Keypoints[2].Value then + return ("ColorSequence.new(%s)"):format(repr(v.Keypoints[1].Value, reprSettings)) + else + return ("ColorSequence.new(%s, %s)"):format( + repr(v.Keypoints[1].Value, reprSettings), + repr(v.Keypoints[2].Value, reprSettings) + ) + end + end + elseif typeof(v) == "ColorSequenceKeypoint" then + return ("ColorSequenceKeypoint.new(%d, %s)"):format(v.Time, repr(v.Value, reprSettings)) + elseif typeof(v) == "DockWidgetPluginGuiInfo" then + return ("DockWidgetPluginGuiInfo.new(%s, %s, %s, %s, %s, %s, %s)"):format( + repr(v.InitialDockState, reprSettings), + repr(v.InitialEnabled, reprSettings), + repr(v.InitialEnabledShouldOverrideRestore, reprSettings), + repr(v.FloatingXSize, reprSettings), + repr(v.FloatingYSize, reprSettings), + repr(v.MinWidth, reprSettings), + repr(v.MinHeight, reprSettings) + ) + elseif typeof(v) == "Enums" then + return "Enums" + elseif typeof(v) == "Enum" then + return ("Enum.%s"):format(tostring(v)) + elseif typeof(v) == "EnumItem" then + return ("Enum.%s.%s"):format(tostring(v.EnumType), v.Name) + elseif typeof(v) == "Faces" then + local s = {} + for _, enumItem in pairs(Enum.NormalId:GetEnumItems()) do + if v[enumItem.Name] then + table.insert(s, repr(enumItem, reprSettings)) + end + end + return ("Faces.new(%s)"):format(table.concat(s, ", ")) + elseif typeof(v) == "NumberRange" then + if v.Min == v.Max then + return ("NumberRange.new(%d)"):format(v.Min) + else + return ("NumberRange.new(%d, %d)"):format(v.Min, v.Max) + end + elseif typeof(v) == "NumberSequence" then + if #v.Keypoints > 2 then + return ("NumberSequence.new(%s)"):format(repr(v.Keypoints, reprSettings)) + else + if v.Keypoints[1].Value == v.Keypoints[2].Value then + return ("NumberSequence.new(%d)"):format(v.Keypoints[1].Value) + else + return ("NumberSequence.new(%d, %d)"):format(v.Keypoints[1].Value, v.Keypoints[2].Value) + end + end + elseif typeof(v) == "NumberSequenceKeypoint" then + if v.Envelope ~= 0 then + return ("NumberSequenceKeypoint.new(%d, %d, %d)"):format(v.Time, v.Value, v.Envelope) + else + return ("NumberSequenceKeypoint.new(%d, %d)"):format(v.Time, v.Value) + end + elseif typeof(v) == "PathWaypoint" then + return ("PathWaypoint.new(%s, %s)"):format( + repr(v.Position, reprSettings), + repr(v.Action, reprSettings) + ) + elseif typeof(v) == "PhysicalProperties" then + return ("PhysicalProperties.new(%d, %d, %d, %d, %d)"):format( + v.Density, v.Friction, v.Elasticity, v.FrictionWeight, v.ElasticityWeight + ) + elseif typeof(v) == "Random" then + return "" + elseif typeof(v) == "Ray" then + return ("Ray.new(%s, %s)"):format( + repr(v.Origin, reprSettings), + repr(v.Direction, reprSettings) + ) + elseif typeof(v) == "RBXScriptConnection" then + return "" + elseif typeof(v) == "RBXScriptSignal" then + return "" + elseif typeof(v) == "Rect" then + return ("Rect.new(%d, %d, %d, %d)"):format( + v.Min.X, v.Min.Y, v.Max.X, v.Max.Y + ) + elseif typeof(v) == "Region3" then + local min = v.CFrame.p + v.Size * -.5 + local max = v.CFrame.p + v.Size * .5 + return ("Region3.new(%s, %s)"):format( + repr(min, reprSettings), + repr(max, reprSettings) + ) + elseif typeof(v) == "Region3int16" then + return ("Region3int16.new(%s, %s)"):format( + repr(v.Min, reprSettings), + repr(v.Max, reprSettings) + ) + elseif typeof(v) == "TweenInfo" then + return ("TweenInfo.new(%d, %s, %s, %d, %s, %d)"):format( + v.Time, repr(v.EasingStyle, reprSettings), repr(v.EasingDirection, reprSettings), + v.RepeatCount, repr(v.Reverses, reprSettings), v.DelayTime + ) + elseif typeof(v) == "UDim" then + return ("UDim.new(%d, %d)"):format( + v.Scale, v.Offset + ) + elseif typeof(v) == "UDim2" then + return ("UDim2.new(%d, %d, %d, %d)"):format( + v.X.Scale, v.X.Offset, v.Y.Scale, v.Y.Offset + ) + elseif typeof(v) == "Vector2" then + return ("Vector2.new(%d, %d)"):format(v.X, v.Y) + elseif typeof(v) == "Vector2int16" then + return ("Vector2int16.new(%d, %d)"):format(v.X, v.Y) + elseif typeof(v) == "Vector3" then + return ("Vector3.new(%d, %d, %d)"):format(v.X, v.Y, v.Z) + elseif typeof(v) == "Vector3int16" then + return ("Vector3int16.new(%d, %d, %d)"):format(v.X, v.Y, v.Z) + elseif typeof(v) == "DateTime" then + return ("DateTime.fromIsoDate(%q)"):format(v:ToIsoDate()) + else + return "" + end + else + return "<" .. type(v) .. ">" + end + end + + return repr +end + +local repr = loadRepr() + +return function(options) + print(('[plugin] create with: %s'):format(repr(options))) + options.extensions = options.extensions or {} + + return { + name = 'load-as-stringvalue', + middleware = function(id) + print('[plugin] get middleware for ', id) + local idExt = id:match('%.(%w+)$') + for _, ext in next, options.extensions do + if ext == idExt then + print('[plugin] matched') + return 'json_model' + end + end + print('[plugin] skipping') + end, + load = function(id) + print('[plugin] get middleware for ', id) + local idExt = id:match('%.(%w+)$') + for _, ext in next, options.extensions do + if ext == idExt then + print('[plugin] matched') + local file = io.open(id, 'r') + local source = file:read('a') + file:close() + return ('{"ClassName": "StringValue", "Properties": { "Value": "%s" }}'):format(source) + end + end + print('[plugin] skipping') + end + } +end diff --git a/test-projects/plugins/src/hello.md b/test-projects/plugins/src/hello.md new file mode 100644 index 000000000..234674249 --- /dev/null +++ b/test-projects/plugins/src/hello.md @@ -0,0 +1,3 @@ +# Markdown + +Woo! diff --git a/test-projects/plugins/src/hello.moon b/test-projects/plugins/src/hello.moon deleted file mode 100644 index ddb781cf6..000000000 --- a/test-projects/plugins/src/hello.moon +++ /dev/null @@ -1 +0,0 @@ -print 'Hello, world!' \ No newline at end of file diff --git a/test-projects/plugins/test-plugin.lua b/test-projects/plugins/test-plugin.lua deleted file mode 100644 index 147463cb2..000000000 --- a/test-projects/plugins/test-plugin.lua +++ /dev/null @@ -1,20 +0,0 @@ -print("test-plugin initializing...") - -return function(nextDispatch, entry) - if entry:isDirectory() then - return nextDispatch(entry) - end - - local name = entry:fileName() - local instanceName = name:match("(.-)%.moon$") - - if instanceName == nil then - return nextDispatch(entry) - end - - return rojo.instance({ - Name = instanceName, - ClassName = "ModuleScript", - Source = compileMoonScript(entry:contents()), - }) -end \ No newline at end of file From daf1b57b77646e9a959aab1c47435e8a57f7d7e0 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 10:18:04 -0500 Subject: [PATCH 07/22] added options support to plugin descriptions, error handling to plugin loading --- src/lua_ast.rs | 7 +++ src/project.rs | 13 +++++- src/serve_session.rs | 52 +++++++++++++++++++--- test-projects/plugins/default.project.json | 7 ++- 4 files changed, 70 insertions(+), 9 deletions(-) diff --git a/src/lua_ast.rs b/src/lua_ast.rs index c31b609f9..088f9ed31 100644 --- a/src/lua_ast.rs +++ b/src/lua_ast.rs @@ -98,6 +98,13 @@ impl FmtLua for Expression { } } +impl fmt::Display for Expression { + fn fmt(&self, output: &mut fmt::Formatter) -> fmt::Result { + let mut stream = LuaStream::new(output); + FmtLua::fmt_lua(self, &mut stream) + } +} + impl From for Expression { fn from(value: String) -> Self { Self::String(value) diff --git a/src/project.rs b/src/project.rs index 531abaa56..dfb8857fb 100644 --- a/src/project.rs +++ b/src/project.rs @@ -31,6 +31,16 @@ enum Error { }, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum PluginDescription { + Source(String), + SourceWithOptions { + source: String, + options: serde_json::Value, + }, +} + /// Contains all of the configuration for a Rojo-managed project. /// /// Project files are stored in `.project.json` files. @@ -40,9 +50,8 @@ pub struct Project { /// The name of the top-level instance described by the project. pub name: String, - // TODO: Support objects as well as strings for plugins (containing 'source' and 'options' properties) #[serde(default = "Vec::new", skip_serializing_if = "Vec::is_empty")] - pub plugins: Vec, + pub plugins: Vec, /// The tree of instances described by this project. Projects always /// describe at least one instance. diff --git a/src/serve_session.rs b/src/serve_session.rs index cd3c2fdbc..b4d1e5383 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -14,9 +14,10 @@ use thiserror::Error; use crate::{ change_processor::ChangeProcessor, + lua_ast::Expression, message_queue::MessageQueue, plugin_env::PluginEnv, - project::{Project, ProjectError}, + project::{PluginDescription, Project, ProjectError}, session_id::SessionId, snapshot::{ apply_patch_set, compute_patch_set, AppliedPatchSet, InstanceContext, InstanceSnapshot, @@ -25,6 +26,27 @@ use crate::{ snapshot_middleware::snapshot_from_vfs, }; +// TODO: Centralize this (copied from json middleware) +fn json_to_lua_value(value: serde_json::Value) -> Expression { + use serde_json::Value; + + match value { + Value::Null => Expression::Nil, + Value::Bool(value) => Expression::Bool(value), + Value::Number(value) => Expression::Number(value.as_f64().unwrap()), + Value::String(value) => Expression::String(value), + Value::Array(values) => { + Expression::Array(values.into_iter().map(json_to_lua_value).collect()) + } + Value::Object(values) => Expression::table( + values + .into_iter() + .map(|(key, value)| (key.into(), json_to_lua_value(value))) + .collect(), + ), + } +} + /// Contains all of the state for a Rojo serve session. A serve session is used /// when we need to build a Rojo tree and possibly rebuild it when input files /// change. @@ -121,12 +143,24 @@ impl ServeSession { }; let plugin_env = PluginEnv::new(); - plugin_env.init().unwrap(); + match plugin_env.init() { + Ok(_) => (), + Err(e) => return Err(ServeSessionError::Plugin { source: e }), + }; - for plugin in root_project.plugins.iter() { - plugin_env - .load_plugin(plugin, "{extensions = {'.md'}}".to_string()) - .unwrap(); + for plugin_description in root_project.plugins.iter() { + let default_options = "{}".to_string(); + let (plugin_source, plugin_options) = match plugin_description { + PluginDescription::Source(source) => (source, default_options), + PluginDescription::SourceWithOptions { source, options } => { + (source, json_to_lua_value(options.to_owned()).to_string()) + } + }; + + match plugin_env.load_plugin(&plugin_source, plugin_options) { + Ok(_) => (), + Err(e) => return Err(ServeSessionError::Plugin { source: e }), + }; } let mut tree = RojoTree::new(InstanceSnapshot::new()); @@ -250,4 +284,10 @@ pub enum ServeSessionError { #[from] source: anyhow::Error, }, + + #[error(transparent)] + Plugin { + #[from] + source: rlua::Error, + }, } diff --git a/test-projects/plugins/default.project.json b/test-projects/plugins/default.project.json index fb49e4fed..91369d2c4 100644 --- a/test-projects/plugins/default.project.json +++ b/test-projects/plugins/default.project.json @@ -1,6 +1,11 @@ { "name": "plugins", - "plugins": ["load-as-stringvalue.lua"], + "plugins": [ + { + "source": "load-as-stringvalue.lua", + "options": { "extensions": [".md"] } + } + ], "tree": { "$className": "DataModel", "ReplicatedStorage": { From f4c9d2316c40407a231d864610574d17ea019f1e Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 10:23:25 -0500 Subject: [PATCH 08/22] removed duplicate code --- src/plugin_env.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/plugin_env.rs b/src/plugin_env.rs index b8dd016aa..f2265eae6 100644 --- a/src/plugin_env.rs +++ b/src/plugin_env.rs @@ -57,18 +57,6 @@ impl PluginEnv { let plugins_table: Table = globals.get("plugins")?; plugins_table.set(plugins_table.len()? + 1, plugin_instance)?; - let run_plugins = lua_ctx.create_function(|lua_ctx, id: String| { - let plugins: Table = lua_ctx.globals().get("plugins")?; - let id_ref: &str = &id; - for plugin in plugins.sequence_values::
() { - let load: Function = plugin?.get("load")?; - load.call(id_ref)?; - } - - Ok(()) - })?; - globals.set("runPlugins", run_plugins)?; - Ok::<(), rlua::Error>(()) }) } From 3df00711d1fdfb88521dfe197ced457161732092 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 11:42:34 -0500 Subject: [PATCH 09/22] almost got middleware working --- src/change_processor.rs | 20 ++- src/plugin_env.rs | 36 +++++- src/serve_session.rs | 9 +- src/snapshot_middleware/dir.rs | 38 ++++-- src/snapshot_middleware/lua.rs | 8 +- src/snapshot_middleware/mod.rs | 118 +++++++++++++----- src/snapshot_middleware/project.rs | 65 ++++++++-- test-projects/plugins/default.project.json | 2 +- test-projects/plugins/load-as-stringvalue.lua | 8 +- 9 files changed, 237 insertions(+), 67 deletions(-) diff --git a/src/change_processor.rs b/src/change_processor.rs index b7ad8b203..6900e1a54 100644 --- a/src/change_processor.rs +++ b/src/change_processor.rs @@ -10,6 +10,7 @@ use rbx_dom_weak::types::{Ref, Variant}; use crate::{ message_queue::MessageQueue, + plugin_env::PluginEnv, snapshot::{ apply_patch_set, compute_patch_set, AppliedPatchSet, InstigatingSource, PatchSet, RojoTree, }, @@ -49,6 +50,7 @@ impl ChangeProcessor { pub fn start( tree: Arc>, vfs: Arc, + plugin_env: Arc>, message_queue: Arc>, tree_mutation_receiver: Receiver, ) -> Self { @@ -57,6 +59,7 @@ impl ChangeProcessor { let task = JobThreadContext { tree, vfs, + plugin_env, message_queue, }; @@ -108,6 +111,8 @@ struct JobThreadContext { /// A handle to the VFS we're managing. vfs: Arc, + plugin_env: Arc>, + /// Whenever changes are applied to the DOM, we should push those changes /// into this message queue to inform any connected clients. message_queue: Arc>, @@ -125,6 +130,7 @@ impl JobThreadContext { // For a given VFS event, we might have many changes to different parts // of the tree. Calculate and apply all of these changes. let applied_patches = { + let plugin_env = self.plugin_env.lock().unwrap(); let mut tree = self.tree.lock().unwrap(); let mut applied_patches = Vec::new(); @@ -153,7 +159,9 @@ impl JobThreadContext { }; for id in affected_ids { - if let Some(patch) = compute_and_apply_changes(&mut tree, &self.vfs, id) { + if let Some(patch) = + compute_and_apply_changes(&mut tree, &self.vfs, &plugin_env, id) + { applied_patches.push(patch); } } @@ -257,7 +265,12 @@ impl JobThreadContext { } } -fn compute_and_apply_changes(tree: &mut RojoTree, vfs: &Vfs, id: Ref) -> Option { +fn compute_and_apply_changes( + tree: &mut RojoTree, + vfs: &Vfs, + plugin_env: &PluginEnv, + id: Ref, +) -> Option { let metadata = tree .get_metadata(id) .expect("metadata missing for instance present in tree"); @@ -283,7 +296,7 @@ fn compute_and_apply_changes(tree: &mut RojoTree, vfs: &Vfs, id: Ref) -> Option< // path still exists. We can generate a snapshot starting at // that path and use it as the source for our patch. - let snapshot = match snapshot_from_vfs(&metadata.context, &vfs, &path) { + let snapshot = match snapshot_from_vfs(&metadata.context, &vfs, plugin_env, &path) { Ok(Some(snapshot)) => snapshot, Ok(None) => { log::error!( @@ -331,6 +344,7 @@ fn compute_and_apply_changes(tree: &mut RojoTree, vfs: &Vfs, id: Ref) -> Option< instance_name, project_node, &vfs, + plugin_env, parent_class.as_ref().map(|name| name.as_str()), ); diff --git a/src/plugin_env.rs b/src/plugin_env.rs index f2265eae6..276488a8f 100644 --- a/src/plugin_env.rs +++ b/src/plugin_env.rs @@ -1,5 +1,7 @@ use rlua::{Function, Lua, Table}; -use std::fs; +use std::{fs, str::FromStr}; + +use crate::snapshot_middleware::SnapshotMiddleware; pub struct PluginEnv { lua: Lua, @@ -54,6 +56,18 @@ impl PluginEnv { let plugin_options_table: Table = lua_ctx.load(&plugin_options).eval()?; let plugin_instance: Table = create_plugin.call(plugin_options_table)?; + let plugin_name: String = plugin_instance.get("name")?; + // if plugin_name.trim().is_empty() { + // return Err(rlua::Error::ExternalError(Arc::new(std::error::Error( + // "".to_string(), + // )))); + // } + log::trace!( + "Loaded plugin '{}' from source: {}", + plugin_name, + plugin_source + ); + let plugins_table: Table = globals.get("plugins")?; plugins_table.set(plugins_table.len()? + 1, plugin_instance)?; @@ -61,14 +75,26 @@ impl PluginEnv { }) } - pub fn run_plugins(&self, id: String) -> Result<(), rlua::Error> { + pub fn get_middlware(&self, id: String) -> Result, rlua::Error> { self.lua.context(|lua_ctx| { let globals = lua_ctx.globals(); - let run_plugins: Function = globals.get("runPlugins")?; - run_plugins.call(id)?; + let plugins: Table = globals.get("plugins")?; + let id_ref: &str = &id; + for plugin in plugins.sequence_values::
() { + let middleware_fn: Function = plugin?.get("middleware")?; + let middleware_str: Option = middleware_fn.call(id_ref)?; + let middleware_enum = match middleware_str { + Some(str) => SnapshotMiddleware::from_str(&str).ok(), + None => None, + }; + if middleware_enum.is_some() { + println!("{:?}", middleware_enum); + return Ok(middleware_enum); + } + } - Ok::<(), rlua::Error>(()) + Ok(None) }) } } diff --git a/src/serve_session.rs b/src/serve_session.rs index b4d1e5383..8f18c5420 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -109,6 +109,8 @@ pub struct ServeSession { /// A channel to send mutation requests on. These will be handled by the /// ChangeProcessor and trigger changes in the tree. tree_mutation_sender: Sender, + + plugin_env: Arc>, } impl ServeSession { @@ -142,7 +144,7 @@ impl ServeSession { } }; - let plugin_env = PluginEnv::new(); + let mut plugin_env = PluginEnv::new(); match plugin_env.init() { Ok(_) => (), Err(e) => return Err(ServeSessionError::Plugin { source: e }), @@ -170,7 +172,7 @@ impl ServeSession { let instance_context = InstanceContext::default(); log::trace!("Generating snapshot of instances from VFS"); - let snapshot = snapshot_from_vfs(&instance_context, &vfs, &start_path)? + let snapshot = snapshot_from_vfs(&instance_context, &vfs, &plugin_env, &start_path)? .expect("snapshot did not return an instance"); log::trace!("Computing initial patch set"); @@ -185,6 +187,7 @@ impl ServeSession { let tree = Arc::new(Mutex::new(tree)); let message_queue = Arc::new(message_queue); let vfs = Arc::new(vfs); + let plugin_env = Arc::new(Mutex::new(plugin_env)); let (tree_mutation_sender, tree_mutation_receiver) = crossbeam_channel::unbounded(); @@ -192,6 +195,7 @@ impl ServeSession { let change_processor = ChangeProcessor::start( Arc::clone(&tree), Arc::clone(&vfs), + Arc::clone(&plugin_env), Arc::clone(&message_queue), tree_mutation_receiver, ); @@ -205,6 +209,7 @@ impl ServeSession { message_queue, tree_mutation_sender, vfs, + plugin_env, }) } diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index bad35c813..c3514353f 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -2,13 +2,17 @@ use std::path::Path; use memofs::{DirEntry, IoResultExt, Vfs}; -use crate::snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}; +use crate::{ + plugin_env::PluginEnv, + snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, +}; use super::{meta_file::DirectoryMetadata, snapshot_from_vfs}; pub fn snapshot_dir( context: &InstanceContext, vfs: &Vfs, + plugin_env: &PluginEnv, path: &Path, ) -> anyhow::Result> { let passes_filter_rules = |child: &DirEntry| { @@ -27,7 +31,7 @@ pub fn snapshot_dir( continue; } - if let Some(child_snapshot) = snapshot_from_vfs(context, vfs, entry.path())? { + if let Some(child_snapshot) = snapshot_from_vfs(context, vfs, plugin_env, entry.path())? { snapshot_children.push(child_snapshot); } } @@ -86,10 +90,17 @@ mod test { let mut vfs = Vfs::new(imfs); - let instance_snapshot = - snapshot_dir(&InstanceContext::default(), &mut vfs, Path::new("/foo")) - .unwrap() - .unwrap(); + let plugin_env = PluginEnv::new(); + plugin_env.init().unwrap(); + + let instance_snapshot = snapshot_dir( + &InstanceContext::default(), + &mut vfs, + &plugin_env, + Path::new("/foo"), + ) + .unwrap() + .unwrap(); insta::assert_yaml_snapshot!(instance_snapshot); } @@ -107,10 +118,17 @@ mod test { let mut vfs = Vfs::new(imfs); - let instance_snapshot = - snapshot_dir(&InstanceContext::default(), &mut vfs, Path::new("/foo")) - .unwrap() - .unwrap(); + let plugin_env = PluginEnv::new(); + plugin_env.init().unwrap(); + + let instance_snapshot = snapshot_dir( + &InstanceContext::default(), + &mut vfs, + &plugin_env, + Path::new("/foo"), + ) + .unwrap() + .unwrap(); insta::assert_yaml_snapshot!(instance_snapshot); } diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index d11e72c62..d21a549c1 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -4,7 +4,10 @@ use anyhow::Context; use maplit::hashmap; use memofs::{IoResultExt, Vfs}; -use crate::snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}; +use crate::{ + plugin_env::PluginEnv, + snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, +}; use super::{dir::snapshot_dir, meta_file::AdjacentMetadata, util::match_trailing}; @@ -63,10 +66,11 @@ pub fn snapshot_lua( pub fn snapshot_lua_init( context: &InstanceContext, vfs: &Vfs, + plugin_env: &PluginEnv, init_path: &Path, ) -> anyhow::Result> { let folder_path = init_path.parent().unwrap(); - let dir_snapshot = snapshot_dir(context, vfs, folder_path)?.unwrap(); + let dir_snapshot = snapshot_dir(context, vfs, plugin_env, folder_path)?.unwrap(); if dir_snapshot.class_name != "Folder" { anyhow::bail!( diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 36729f8f3..036939ac3 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -17,11 +17,14 @@ mod rbxmx; mod txt; mod util; -use std::path::Path; +use std::{path::Path, str::FromStr}; use memofs::{IoResultExt, Vfs}; -use crate::snapshot::{InstanceContext, InstanceSnapshot}; +use crate::{ + plugin_env::PluginEnv, + snapshot::{InstanceContext, InstanceSnapshot}, +}; use self::{ csv::snapshot_csv, @@ -38,11 +41,44 @@ use self::{ pub use self::project::snapshot_project_node; +#[derive(Debug)] +pub enum SnapshotMiddleware { + Csv, + Dir, + Json, + JsonModel, + Lua, + Project, + Rbxm, + Rbxmx, + Txt, +} + +impl FromStr for SnapshotMiddleware { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "csv" => Ok(SnapshotMiddleware::Csv), + "dir" => Ok(SnapshotMiddleware::Dir), + "json" => Ok(SnapshotMiddleware::Json), + "json_model" => Ok(SnapshotMiddleware::JsonModel), + "lua" => Ok(SnapshotMiddleware::Lua), + "project" => Ok(SnapshotMiddleware::Project), + "rbxm" => Ok(SnapshotMiddleware::Rbxm), + "rbxmx" => Ok(SnapshotMiddleware::Rbxmx), + "txt" => Ok(SnapshotMiddleware::Txt), + _ => Err(format!("Unknown snapshot middleware: {}", s)), + } + } +} + /// The main entrypoint to the snapshot function. This function can be pointed /// at any path and will return something if Rojo knows how to deal with it. pub fn snapshot_from_vfs( context: &InstanceContext, vfs: &Vfs, + plugin_env: &PluginEnv, path: &Path, ) -> anyhow::Result> { let meta = match vfs.metadata(path).with_not_found()? { @@ -53,53 +89,69 @@ pub fn snapshot_from_vfs( if meta.is_dir() { let project_path = path.join("default.project.json"); if vfs.metadata(&project_path).with_not_found()?.is_some() { - return snapshot_project(context, vfs, &project_path); + return snapshot_project(context, vfs, plugin_env, &project_path); } let init_path = path.join("init.lua"); if vfs.metadata(&init_path).with_not_found()?.is_some() { - return snapshot_lua_init(context, vfs, &init_path); + return snapshot_lua_init(context, vfs, plugin_env, &init_path); } let init_path = path.join("init.server.lua"); if vfs.metadata(&init_path).with_not_found()?.is_some() { - return snapshot_lua_init(context, vfs, &init_path); + return snapshot_lua_init(context, vfs, plugin_env, &init_path); } let init_path = path.join("init.client.lua"); if vfs.metadata(&init_path).with_not_found()?.is_some() { - return snapshot_lua_init(context, vfs, &init_path); + return snapshot_lua_init(context, vfs, plugin_env, &init_path); } - snapshot_dir(context, vfs, path) + snapshot_dir(context, vfs, plugin_env, path) } else { - if let Ok(name) = path.file_name_trim_end(".lua") { - match name { - // init scripts are handled elsewhere and should not turn into - // their own children. - "init" | "init.client" | "init.server" => return Ok(None), - - _ => return snapshot_lua(context, vfs, path), - } - } else if path.file_name_ends_with(".project.json") { - return snapshot_project(context, vfs, path); - } else if path.file_name_ends_with(".model.json") { - return snapshot_json_model(context, vfs, path); - } else if path.file_name_ends_with(".meta.json") { - // .meta.json files do not turn into their own instances. - return Ok(None); - } else if path.file_name_ends_with(".json") { - return snapshot_json(context, vfs, path); - } else if path.file_name_ends_with(".csv") { - return snapshot_csv(context, vfs, path); - } else if path.file_name_ends_with(".txt") { - return snapshot_txt(context, vfs, path); - } else if path.file_name_ends_with(".rbxmx") { - return snapshot_rbxmx(context, vfs, path); - } else if path.file_name_ends_with(".rbxm") { - return snapshot_rbxm(context, vfs, path); + let mut middleware = plugin_env.get_middlware(path.to_str().unwrap().to_owned())?; + + if middleware.is_none() { + middleware = if let Ok(name) = path.file_name_trim_end(".lua") { + match name { + "init" | "init.client" | "init.server" => None, + _ => Some(SnapshotMiddleware::Lua), + } + } else if path.file_name_ends_with(".project.json") { + Some(SnapshotMiddleware::Project) + } else if path.file_name_ends_with(".model.json") { + Some(SnapshotMiddleware::JsonModel) + } else if path.file_name_ends_with(".meta.json") { + // .meta.json files do not turn into their own instances. + None + } else if path.file_name_ends_with(".json") { + Some(SnapshotMiddleware::Json) + } else if path.file_name_ends_with(".csv") { + Some(SnapshotMiddleware::Csv) + } else if path.file_name_ends_with(".txt") { + Some(SnapshotMiddleware::Txt) + } else if path.file_name_ends_with(".rbxmx") { + Some(SnapshotMiddleware::Rbxmx) + } else if path.file_name_ends_with(".rbxm") { + Some(SnapshotMiddleware::Rbxm) + } else { + None + }; } - Ok(None) + return match middleware { + Some(x) => match x { + SnapshotMiddleware::Lua => snapshot_lua(context, vfs, path), + SnapshotMiddleware::Project => snapshot_project(context, vfs, plugin_env, path), + SnapshotMiddleware::JsonModel => snapshot_json_model(context, vfs, path), + SnapshotMiddleware::Json => snapshot_json(context, vfs, path), + SnapshotMiddleware::Csv => snapshot_csv(context, vfs, path), + SnapshotMiddleware::Txt => snapshot_txt(context, vfs, path), + SnapshotMiddleware::Rbxmx => snapshot_rbxmx(context, vfs, path), + SnapshotMiddleware::Rbxm => snapshot_rbxm(context, vfs, path), + _ => Ok(None), + }, + None => Ok(None), + }; } } diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index 673def0fd..3b1b98042 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -5,6 +5,7 @@ use memofs::Vfs; use rbx_reflection::ClassTag; use crate::{ + plugin_env::PluginEnv, project::{Project, ProjectNode}, snapshot::{ InstanceContext, InstanceMetadata, InstanceSnapshot, InstigatingSource, PathIgnoreRule, @@ -16,6 +17,7 @@ use super::snapshot_from_vfs; pub fn snapshot_project( context: &InstanceContext, vfs: &Vfs, + plugin_env: &PluginEnv, path: &Path, ) -> anyhow::Result> { let project = Project::load_from_slice(&vfs.read(path)?, path) @@ -32,8 +34,16 @@ pub fn snapshot_project( // TODO: If this project node is a path to an instance that Rojo doesn't // understand, this may panic! - let mut snapshot = - snapshot_project_node(&context, path, &project.name, &project.tree, vfs, None)?.unwrap(); + let mut snapshot = snapshot_project_node( + &context, + path, + &project.name, + &project.tree, + vfs, + plugin_env, + None, + )? + .unwrap(); // Setting the instigating source to the project file path is a little // coarse. @@ -62,6 +72,7 @@ pub fn snapshot_project_node( instance_name: &str, node: &ProjectNode, vfs: &Vfs, + plugin_env: &PluginEnv, parent_class: Option<&str>, ) -> anyhow::Result> { let project_folder = project_path.parent().unwrap(); @@ -86,7 +97,7 @@ pub fn snapshot_project_node( Cow::Borrowed(path) }; - if let Some(snapshot) = snapshot_from_vfs(context, vfs, &path)? { + if let Some(snapshot) = snapshot_from_vfs(context, vfs, plugin_env, &path)? { class_name_from_path = Some(snapshot.class_name); // Properties from the snapshot are pulled in unchanged, and @@ -182,6 +193,7 @@ pub fn snapshot_project_node( child_name, child_project_node, vfs, + plugin_env, Some(&class_name), )? { children.push(child); @@ -304,10 +316,17 @@ mod test { let mut vfs = Vfs::new(imfs); - let instance_snapshot = - snapshot_project(&InstanceContext::default(), &mut vfs, Path::new("/foo")) - .expect("snapshot error") - .expect("snapshot returned no instances"); + let plugin_env = PluginEnv::new(); + plugin_env.init().unwrap(); + + let instance_snapshot = snapshot_project( + &InstanceContext::default(), + &mut vfs, + &plugin_env, + Path::new("/foo"), + ) + .expect("snapshot error") + .expect("snapshot returned no instances"); insta::assert_yaml_snapshot!(instance_snapshot); } @@ -334,9 +353,13 @@ mod test { let mut vfs = Vfs::new(imfs); + let plugin_env = PluginEnv::new(); + plugin_env.init().unwrap(); + let instance_snapshot = snapshot_project( &InstanceContext::default(), &mut vfs, + &plugin_env, Path::new("/foo/hello.project.json"), ) .expect("snapshot error") @@ -372,9 +395,13 @@ mod test { let mut vfs = Vfs::new(imfs); + let plugin_env = PluginEnv::new(); + plugin_env.init().unwrap(); + let instance_snapshot = snapshot_project( &InstanceContext::default(), &mut vfs, + &plugin_env, Path::new("/foo.project.json"), ) .expect("snapshot error") @@ -408,9 +435,13 @@ mod test { let mut vfs = Vfs::new(imfs); + let plugin_env = PluginEnv::new(); + plugin_env.init().unwrap(); + let instance_snapshot = snapshot_project( &InstanceContext::default(), &mut vfs, + &plugin_env, Path::new("/foo.project.json"), ) .expect("snapshot error") @@ -445,9 +476,13 @@ mod test { let mut vfs = Vfs::new(imfs); + let plugin_env = PluginEnv::new(); + plugin_env.init().unwrap(); + let instance_snapshot = snapshot_project( &InstanceContext::default(), &mut vfs, + &plugin_env, Path::new("/foo.project.json"), ) .expect("snapshot error") @@ -479,9 +514,13 @@ mod test { let mut vfs = Vfs::new(imfs); + let plugin_env = PluginEnv::new(); + plugin_env.init().unwrap(); + let instance_snapshot = snapshot_project( &InstanceContext::default(), &mut vfs, + &plugin_env, Path::new("/foo/default.project.json"), ) .expect("snapshot error") @@ -520,9 +559,13 @@ mod test { let mut vfs = Vfs::new(imfs); + let plugin_env = PluginEnv::new(); + plugin_env.init().unwrap(); + let instance_snapshot = snapshot_project( &InstanceContext::default(), &mut vfs, + &plugin_env, Path::new("/foo/default.project.json"), ) .expect("snapshot error") @@ -565,9 +608,13 @@ mod test { let mut vfs = Vfs::new(imfs); + let plugin_env = PluginEnv::new(); + plugin_env.init().unwrap(); + let instance_snapshot = snapshot_project( &InstanceContext::default(), &mut vfs, + &plugin_env, Path::new("/foo/default.project.json"), ) .expect("snapshot error") @@ -615,9 +662,13 @@ mod test { let mut vfs = Vfs::new(imfs); + let plugin_env = PluginEnv::new(); + plugin_env.init().unwrap(); + let instance_snapshot = snapshot_project( &InstanceContext::default(), &mut vfs, + &plugin_env, Path::new("/foo/default.project.json"), ) .expect("snapshot error") diff --git a/test-projects/plugins/default.project.json b/test-projects/plugins/default.project.json index 91369d2c4..d80bdd8ae 100644 --- a/test-projects/plugins/default.project.json +++ b/test-projects/plugins/default.project.json @@ -3,7 +3,7 @@ "plugins": [ { "source": "load-as-stringvalue.lua", - "options": { "extensions": [".md"] } + "options": { "extensions": ["md"] } } ], "tree": { diff --git a/test-projects/plugins/load-as-stringvalue.lua b/test-projects/plugins/load-as-stringvalue.lua index f9707ae27..a15d879ee 100644 --- a/test-projects/plugins/load-as-stringvalue.lua +++ b/test-projects/plugins/load-as-stringvalue.lua @@ -290,22 +290,22 @@ return function(options) return { name = 'load-as-stringvalue', middleware = function(id) - print('[plugin] get middleware for ', id) + print(('[plugin] middleware: %s'):format(id)) local idExt = id:match('%.(%w+)$') for _, ext in next, options.extensions do if ext == idExt then - print('[plugin] matched') + print(('[plugin] matched: %s'):format(ext)) return 'json_model' end end print('[plugin] skipping') end, load = function(id) - print('[plugin] get middleware for ', id) + print(('[plugin] load: %s'):format(id)) local idExt = id:match('%.(%w+)$') for _, ext in next, options.extensions do if ext == idExt then - print('[plugin] matched') + print('[plugin] matched: %s') local file = io.open(id, 'r') local source = file:read('a') file:close() From f44755713ef8fe4c79330d517a77bbe949bfd429 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 12:18:33 -0500 Subject: [PATCH 10/22] hacked it to make it work lol --- src/lib.rs | 1 + src/load_file.rs | 24 +++++++++++++++++++ src/plugin_env.rs | 23 ++++++++++++++---- src/serve_session.rs | 2 +- src/snapshot_middleware/json_model.rs | 14 +++++++++-- src/snapshot_middleware/mod.rs | 6 +++-- test-projects/plugins/load-as-stringvalue.lua | 10 ++++---- 7 files changed, 65 insertions(+), 15 deletions(-) create mode 100644 src/load_file.rs diff --git a/src/lib.rs b/src/lib.rs index 92b719ac6..8c1291e21 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,7 @@ mod tree_view; mod auth_cookie; mod change_processor; mod glob; +mod load_file; mod lua_ast; mod message_queue; mod multimap; diff --git a/src/load_file.rs b/src/load_file.rs new file mode 100644 index 000000000..327ec5573 --- /dev/null +++ b/src/load_file.rs @@ -0,0 +1,24 @@ +use anyhow::Context; +use memofs::Vfs; +use std::{path::Path, str, sync::Arc}; + +use crate::plugin_env::PluginEnv; + +pub fn load_file( + vfs: &Vfs, + plugin_env: &PluginEnv, + path: &Path, +) -> Result>, anyhow::Error> { + let contents = vfs.read(path)?; + let contents_str = str::from_utf8(&contents) + .with_context(|| format!("File was not valid UTF-8: {}", path.display()))?; + + let plugin_result = plugin_env.load(path.to_str().unwrap(), contents_str); + match plugin_result { + Ok(Some(data)) => return Ok(Arc::new(data.as_bytes().to_vec())), + Ok(None) => {} + Err(_) => {} + } + + return Ok(contents); +} diff --git a/src/plugin_env.rs b/src/plugin_env.rs index 276488a8f..8f97be4f2 100644 --- a/src/plugin_env.rs +++ b/src/plugin_env.rs @@ -75,21 +75,19 @@ impl PluginEnv { }) } - pub fn get_middlware(&self, id: String) -> Result, rlua::Error> { + pub fn middleware(&self, id: &str) -> Result, rlua::Error> { self.lua.context(|lua_ctx| { let globals = lua_ctx.globals(); let plugins: Table = globals.get("plugins")?; - let id_ref: &str = &id; for plugin in plugins.sequence_values::
() { let middleware_fn: Function = plugin?.get("middleware")?; - let middleware_str: Option = middleware_fn.call(id_ref)?; + let middleware_str: Option = middleware_fn.call(id)?; let middleware_enum = match middleware_str { Some(str) => SnapshotMiddleware::from_str(&str).ok(), None => None, }; if middleware_enum.is_some() { - println!("{:?}", middleware_enum); return Ok(middleware_enum); } } @@ -97,4 +95,21 @@ impl PluginEnv { Ok(None) }) } + + pub fn load(&self, id: &str, data: &str) -> Result, rlua::Error> { + self.lua.context(|lua_ctx| { + let globals = lua_ctx.globals(); + + let plugins: Table = globals.get("plugins")?; + for plugin in plugins.sequence_values::
() { + let load_fn: Function = plugin?.get("load")?; + let load_str: Option = load_fn.call((id, data))?; + if load_str.is_some() { + return Ok(load_str); + } + } + + Ok(None) + }) + } } diff --git a/src/serve_session.rs b/src/serve_session.rs index 8f18c5420..a83cc885f 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -144,7 +144,7 @@ impl ServeSession { } }; - let mut plugin_env = PluginEnv::new(); + let plugin_env = PluginEnv::new(); match plugin_env.init() { Ok(_) => (), Err(e) => return Err(ServeSessionError::Plugin { source: e }), diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 520710312..07a657415 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -5,6 +5,8 @@ use memofs::Vfs; use serde::Deserialize; use crate::{ + load_file::load_file, + plugin_env::PluginEnv, resolution::UnresolvedValue, snapshot::{InstanceContext, InstanceSnapshot}, }; @@ -14,13 +16,17 @@ use super::util::PathExt; pub fn snapshot_json_model( context: &InstanceContext, vfs: &Vfs, + plugin_env: &PluginEnv, path: &Path, ) -> anyhow::Result> { - let name = path.file_name_trim_end(".model.json")?; + // let name = path.file_name_trim_end(".model.json")?; + let name = path.file_name().and_then(|s| s.to_str()).unwrap(); - let contents = vfs.read(path)?; + let contents = load_file(vfs, plugin_env, path)?; + // let contents = vfs.read(path)?; let contents_str = str::from_utf8(&contents) .with_context(|| format!("File was not valid UTF-8: {}", path.display()))?; + println!("{}", contents_str); if contents_str.trim().is_empty() { return Ok(None); @@ -132,9 +138,13 @@ mod test { let mut vfs = Vfs::new(imfs); + let plugin_env = PluginEnv::new(); + plugin_env.init().unwrap(); + let instance_snapshot = snapshot_json_model( &InstanceContext::default(), &mut vfs, + &plugin_env, Path::new("/foo.model.json"), ) .unwrap() diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 036939ac3..76d9b81f8 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -109,7 +109,7 @@ pub fn snapshot_from_vfs( snapshot_dir(context, vfs, plugin_env, path) } else { - let mut middleware = plugin_env.get_middlware(path.to_str().unwrap().to_owned())?; + let mut middleware = plugin_env.middleware(path.to_str().unwrap())?; if middleware.is_none() { middleware = if let Ok(name) = path.file_name_trim_end(".lua") { @@ -143,7 +143,9 @@ pub fn snapshot_from_vfs( Some(x) => match x { SnapshotMiddleware::Lua => snapshot_lua(context, vfs, path), SnapshotMiddleware::Project => snapshot_project(context, vfs, plugin_env, path), - SnapshotMiddleware::JsonModel => snapshot_json_model(context, vfs, path), + SnapshotMiddleware::JsonModel => { + snapshot_json_model(context, vfs, plugin_env, path) + } SnapshotMiddleware::Json => snapshot_json(context, vfs, path), SnapshotMiddleware::Csv => snapshot_csv(context, vfs, path), SnapshotMiddleware::Txt => snapshot_txt(context, vfs, path), diff --git a/test-projects/plugins/load-as-stringvalue.lua b/test-projects/plugins/load-as-stringvalue.lua index a15d879ee..3cfe9a172 100644 --- a/test-projects/plugins/load-as-stringvalue.lua +++ b/test-projects/plugins/load-as-stringvalue.lua @@ -300,16 +300,14 @@ return function(options) end print('[plugin] skipping') end, - load = function(id) + load = function(id, data) print(('[plugin] load: %s'):format(id)) local idExt = id:match('%.(%w+)$') for _, ext in next, options.extensions do if ext == idExt then - print('[plugin] matched: %s') - local file = io.open(id, 'r') - local source = file:read('a') - file:close() - return ('{"ClassName": "StringValue", "Properties": { "Value": "%s" }}'):format(source) + print(('[plugin] matched: %s'):format(ext)) + local encoded = data:gsub('\n', '\\n') + return ('{"ClassName": "StringValue", "Properties": { "Value": "%s" }}'):format(encoded) end end print('[plugin] skipping') From 7457120e230ee14c5e020378d619e988a1379734 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 12:32:40 -0500 Subject: [PATCH 11/22] rename data to contents, update plugin implementations in design doc --- PLUGIN-DESIGN.md | 106 +++++++++++++----- src/plugin_env.rs | 4 +- test-projects/plugins/load-as-stringvalue.lua | 4 +- 3 files changed, 85 insertions(+), 29 deletions(-) diff --git a/PLUGIN-DESIGN.md b/PLUGIN-DESIGN.md index 291ff9f60..b0e9468c6 100644 --- a/PLUGIN-DESIGN.md +++ b/PLUGIN-DESIGN.md @@ -181,37 +181,93 @@ return function(options) end ``` -### Custom file types +### Markdown parser ```lua +-- Convert markdown to roblox rich text format implementation here + return function(options) - return { - name = "markdown-to-stringvalue", - middleware = function(id) - if id:match('%.md$') then - return 'json_model' - end - end, - load = function(id) - if id:match('%.md$') then - local file = io.open(id, 'r') - local source = file:read('a') - file:close() - return ('{"ClassName": "StringValue", "Properties": { "Value": "%s" }}'):format(source) - - -- If we had a library of common functions we could use this nicer syntax: - --[[return rojo.toJson({ - ClassName = 'StringValue', - Properties = { - Value = source - } - })]] - end - end - } + return { + name = 'markdown-to-richtext', + middleware = function(id) + if id:match('%.md$') then + return 'json_model' + end + end, + load = function(id, contents) + if id:match('%.md$') then + local frontmatter = parseFrontmatter(contents) + local richText = markdownToRichText(contents) + local className = frontmatter.className or 'StringValue' + local property = frontmatter.property or 'Value' + return ('{"ClassName": "%s", "Properties": { "%s": "%s" }}') + :format(className, property, richText) + + --[[ + With rojo plugin library: + + return rojo.toJson({ + ClassName = className, + Properties = { + [property] = richText + } + }) + ]] + end + end + } end ``` +### Load custom files as StringValue instances + +```lua +return function(options) + options.extensions = options.extensions or {} + + return { + name = 'load-as-stringvalue', + middleware = function(id) + local idExt = id:match('%.(%w+)$') + for _, ext in next, options.extensions do + if ext == idExt then + return 'json_model' + end + end + end, + load = function(id, contents) + local idExt = id:match('%.(%w+)$') + for _, ext in next, options.extensions do + if ext == idExt then + local encoded = contents:gsub('\n', '\\n') + return ('{"ClassName": "StringValue", "Properties": { "Value": "%s" }}'):format(encoded) + + --[[ + With rojo plugin library: + + return rojo.toJson({ + ClassName = 'StringValue', + Properties = { + Value = encoded + } + }) + ]] + end + end + end + } +end +``` + +```json +// default.project.json +{ + "plugins": [ + { "source": "load-as-stringvalue.lua", "options": { "extensions": {"md", "data.json"} }} + ] +} +``` + ### Remote file requires ```lua diff --git a/src/plugin_env.rs b/src/plugin_env.rs index 8f97be4f2..e9c680b15 100644 --- a/src/plugin_env.rs +++ b/src/plugin_env.rs @@ -96,14 +96,14 @@ impl PluginEnv { }) } - pub fn load(&self, id: &str, data: &str) -> Result, rlua::Error> { + pub fn load(&self, id: &str, contents: &str) -> Result, rlua::Error> { self.lua.context(|lua_ctx| { let globals = lua_ctx.globals(); let plugins: Table = globals.get("plugins")?; for plugin in plugins.sequence_values::
() { let load_fn: Function = plugin?.get("load")?; - let load_str: Option = load_fn.call((id, data))?; + let load_str: Option = load_fn.call((id, contents))?; if load_str.is_some() { return Ok(load_str); } diff --git a/test-projects/plugins/load-as-stringvalue.lua b/test-projects/plugins/load-as-stringvalue.lua index 3cfe9a172..7c7036ef8 100644 --- a/test-projects/plugins/load-as-stringvalue.lua +++ b/test-projects/plugins/load-as-stringvalue.lua @@ -300,13 +300,13 @@ return function(options) end print('[plugin] skipping') end, - load = function(id, data) + load = function(id, contents) print(('[plugin] load: %s'):format(id)) local idExt = id:match('%.(%w+)$') for _, ext in next, options.extensions do if ext == idExt then print(('[plugin] matched: %s'):format(ext)) - local encoded = data:gsub('\n', '\\n') + local encoded = contents:gsub('\n', '\\n') return ('{"ClassName": "StringValue", "Properties": { "Value": "%s" }}'):format(encoded) end end From 8c3113cfce4bb11b6561bc81afc6b13f78628ee0 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 12:55:56 -0500 Subject: [PATCH 12/22] added links to requests for plugin features --- PLUGIN-DESIGN.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/PLUGIN-DESIGN.md b/PLUGIN-DESIGN.md index b0e9468c6..435dd01ce 100644 --- a/PLUGIN-DESIGN.md +++ b/PLUGIN-DESIGN.md @@ -135,6 +135,11 @@ shown using the API. ### MoonScript transformation +Requested by: + +- @Airwarfare in [#170](https://github.com/rojo-rbx/rojo/issues/170) +- @dimitriye98 in [#55](https://github.com/rojo-rbx/rojo/issues/55#issuecomment-402616429) (comment) + ```lua local parse = require 'moonscript.parse' local compile = require 'moonscript.compile' @@ -163,6 +168,11 @@ end ### Obfuscation/minifier transformation +Requested by: + +- @cmumme in [#55](https://github.com/rojo-rbx/rojo/issues/55#issuecomment-794801625) (comment) +- @blake-mealey in [#382](https://github.com/rojo-rbx/rojo/issues/382) + ```lua local minifier = require 'minifier.lua' @@ -181,10 +191,10 @@ return function(options) end ``` -### Markdown parser +### Markdown to Roblox rich text ```lua --- Convert markdown to roblox rich text format implementation here +-- Convert markdown to Roblox rich text format implementation here return function(options) return { @@ -221,6 +231,11 @@ end ### Load custom files as StringValue instances +Requested by: + +- @rfrey-rbx in [#406](https://github.com/rojo-rbx/rojo/issues/406) +- @Quenty in [#148](https://github.com/rojo-rbx/rojo/issues/148) + ```lua return function(options) options.extensions = options.extensions or {} @@ -298,6 +313,10 @@ end ### File system requires +Requested by: + +- @blake-mealey in [#382](https://github.com/rojo-rbx/rojo/issues/382) + ```lua -- lua parsing/writing implementation here From 551997b5bfac195cdb6f77ed0fed75f75bbccfc6 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 13:30:18 -0500 Subject: [PATCH 13/22] add rojo plugin library to design doc --- PLUGIN-DESIGN.md | 180 ++++++++++++++++++++++++----------------------- 1 file changed, 92 insertions(+), 88 deletions(-) diff --git a/PLUGIN-DESIGN.md b/PLUGIN-DESIGN.md index 435dd01ce..027d47de8 100644 --- a/PLUGIN-DESIGN.md +++ b/PLUGIN-DESIGN.md @@ -90,6 +90,7 @@ the user in the project file. The plugin environment is created in the following way: 1. Create a new Lua context. +1. Add a global `rojo` table which is the entry point to the [Plugin library](#plugin-library) 1. Initialize an empty `_G.plugins` table. 1. For each plugin description in the project file: 1. Convert the plugin options from the project file from JSON to a Lua table. @@ -128,6 +129,20 @@ message. - **Optional**: Takes a file path and returns the file contents that should be interpreted by Rojo. The first plugin to return a non-nil value per id wins. +## Plugin library + +Accessible via the `rojo` global, the plugin library offers helper methods for plugins: + +- `toJson(value: any) -> string` + - Converts a Lua value to a JSON string. +- `readFile(id: string) -> string` + - Reads the contents of a file from a file path using the VFS. Plugins should always read + files via this method rather than directly from the file system. +- `getExtension(id: string) -> boolean` + - Returns the file extension of the file path. +- `hasExtension(id: string, ext: string) -> boolean` + - Checks if a file path has the provided extension. + ## Use case analyses To demonstrate the effectiveness of this API, pseudo-implementations for a variety of use-cases are @@ -148,16 +163,14 @@ return function(options) return { name = "moonscript", load = function(id) - if id:match('%.lua$') then - local file = io.open(id, 'r') - local source = file:read('a') - file:close() + if rojo.hasExtension(id, 'lua') then + local contents = rojo.readFile(id) - local tree, err = parse.string(source) + local tree, err = parse.string(contents) assert(tree, err) local lua, err, pos = compile.tree(tree) - if not lua then error(compile.format_error(err, pos, source)) end + if not lua then error(compile.format_error(err, pos, contents)) end return lua end @@ -180,11 +193,9 @@ return function(options) return { name = "minifier", load = function(id) - if id:match('%.lua$') then - local file = io.open(id, 'r') - local source = file:read('a') - file:close() - return minifier(source) + if rojo.hasExtension(id, 'lua') then + local contents = rojo.readFile(id) + return minifier(contents) end end } @@ -200,29 +211,26 @@ return function(options) return { name = 'markdown-to-richtext', middleware = function(id) - if id:match('%.md$') then + if rojo.hasExtension(id, 'md') then return 'json_model' end end, - load = function(id, contents) - if id:match('%.md$') then + load = function(id) + if rojo.hasExtension(id, 'md') then + local contents = rojo.readFile(id) + local frontmatter = parseFrontmatter(contents) - local richText = markdownToRichText(contents) local className = frontmatter.className or 'StringValue' local property = frontmatter.property or 'Value' - return ('{"ClassName": "%s", "Properties": { "%s": "%s" }}') - :format(className, property, richText) - - --[[ - With rojo plugin library: - - return rojo.toJson({ - ClassName = className, - Properties = { - [property] = richText - } - }) - ]] + + local richText = markdownToRichText(contents) + + return rojo.toJson({ + ClassName = className, + Properties = { + [property] = richText + } + }) end end } @@ -243,30 +251,26 @@ return function(options) return { name = 'load-as-stringvalue', middleware = function(id) - local idExt = id:match('%.(%w+)$') + local idExt = rojo.getExtension(id) for _, ext in next, options.extensions do if ext == idExt then return 'json_model' end end end, - load = function(id, contents) - local idExt = id:match('%.(%w+)$') + load = function(id) + local idExt = rojo.getExtension(id) for _, ext in next, options.extensions do if ext == idExt then - local encoded = contents:gsub('\n', '\\n') - return ('{"ClassName": "StringValue", "Properties": { "Value": "%s" }}'):format(encoded) - - --[[ - With rojo plugin library: - - return rojo.toJson({ - ClassName = 'StringValue', - Properties = { - Value = encoded - } - }) - ]] + local contents = rojo.readFile(id) + local jsonEncoded = contents:gsub('\r', '\\r'):gsub('\n', '\\n') + + return rojo.toJson({ + ClassName = 'StringValue', + Properties = { + Value = jsonEncoded + } + }) end end end @@ -283,34 +287,6 @@ end } ``` -### Remote file requires - -```lua --- download/caching implementation inline here --- this one is not really working even from a pseudo-implementation perspective - -return function(options) - return { - name = "remote-require", - resolve = function(id) - if id:match('^https?://.*%.lua$') then - local cachedId = fromCache(id) - return cachedId or nil - end - end, - load = function(id) - if id:match('^https?://.*%.lua$') then - local cachedId = downloadAndCache(id) - local file = io.open(cachedId, 'r') - local source = file:read('a') - file:close() - return source - end - end - } -end -``` - ### File system requires Requested by: @@ -329,33 +305,61 @@ return function(options) project = newProject end, load = function(id) - if id:match('%.lua$') then - local file = io.open(id, 'r') - local source = file:read('a') - file:close() + if rojo.hasExtension(id, 'lua') then + local contents = rojo.readFile(id) -- This function will look for require 'file/path' statements in the source and replace -- them with Roblox require(instance.path) statements based on the project's configuration -- (where certain file paths are mounted) - return replaceRequires(source, project) + return replaceRequires(contents, project) end end } end ``` -## Implementation priority + -## Concerns +## Implementation priority -TODO: Implement a proposal for a rojo plugin library +This proposal could be split up into milestones without all the features being present till the end +to simplify development. Here is a proposal for the order to implement each milestone: -- Some operations will be common in plugins and a set of standardized functions may be helpful, - for example reading files and checking file extensions. This could be provided as a global - library injected in the initialization stage of the Lua context (e.g. - `rojo.fileExtensionMatches(id, ext)`, `rojo.loadFile(id)`, `rojo.toJson(value)`). +1. Loading plugins from local paths +1. Minimum Rojo plugin library (`readFile`) +1. Calling hooks at the appropriate time + 1. Start with `middleware`, `load` + 1. Add `syncStart`, `syncEnd`, `resolve` + 1. Add `projectDescription` +1. Full Rojo plugin library +1. Loading plugins from remote repos From 90826c9446021418014003eeb68fe617bf472336 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 14:12:18 -0500 Subject: [PATCH 14/22] improve error handling in plugin loading --- src/plugin_env.rs | 57 +++- src/snapshot_middleware/json_model.rs | 1 - test-projects/plugins/load-as-stringvalue.lua | 293 +----------------- .../plugins/src/{hello.md => Hello.md} | 2 +- 4 files changed, 63 insertions(+), 290 deletions(-) rename test-projects/plugins/src/{hello.md => Hello.md} (60%) diff --git a/src/plugin_env.rs b/src/plugin_env.rs index e9c680b15..47f5a1ced 100644 --- a/src/plugin_env.rs +++ b/src/plugin_env.rs @@ -51,17 +51,54 @@ impl PluginEnv { self.lua.context(|lua_ctx| { let globals = lua_ctx.globals(); - let create_plugin: Function = lua_ctx.load(plugin_lua).eval()?; - - let plugin_options_table: Table = lua_ctx.load(&plugin_options).eval()?; - let plugin_instance: Table = create_plugin.call(plugin_options_table)?; + let create_plugin_fn: Option = + lua_ctx.load(plugin_lua).set_name(plugin_source)?.eval()?; + let create_plugin_fn = match create_plugin_fn { + Some(v) => v, + None => { + return Err(rlua::Error::RuntimeError( + format!( + "plugin from source '{}' did not return a creation function.", + plugin_source + ) + .to_string(), + )) + } + }; + + let plugin_options_table: Table = lua_ctx + .load(&plugin_options) + .set_name("plugin options")? + .eval()?; + + let plugin_instance: Option
= create_plugin_fn.call(plugin_options_table)?; + let plugin_instance = match plugin_instance { + Some(v) => v, + None => { + return Err(rlua::Error::RuntimeError( + format!( + "creation function for plugin from source '{}' did not return a plugin instance.", + plugin_source + ) + .to_string(), + )) + } + }; + + let plugin_name: Option = plugin_instance.get("name")?; + let plugin_name = match plugin_name.unwrap_or("".to_owned()) { + v if v.trim().is_empty() => { + return Err(rlua::Error::RuntimeError( + format!( + "plugin instance for plugin from source '{}' did not have a name.", + plugin_source + ) + .to_string(), + )) + }, + v => v + }; - let plugin_name: String = plugin_instance.get("name")?; - // if plugin_name.trim().is_empty() { - // return Err(rlua::Error::ExternalError(Arc::new(std::error::Error( - // "".to_string(), - // )))); - // } log::trace!( "Loaded plugin '{}' from source: {}", plugin_name, diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 07a657415..97347d8b0 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -26,7 +26,6 @@ pub fn snapshot_json_model( // let contents = vfs.read(path)?; let contents_str = str::from_utf8(&contents) .with_context(|| format!("File was not valid UTF-8: {}", path.display()))?; - println!("{}", contents_str); if contents_str.trim().is_empty() { return Ok(None); diff --git a/test-projects/plugins/load-as-stringvalue.lua b/test-projects/plugins/load-as-stringvalue.lua index 7c7036ef8..2c270b463 100644 --- a/test-projects/plugins/load-as-stringvalue.lua +++ b/test-projects/plugins/load-as-stringvalue.lua @@ -1,290 +1,27 @@ print('[plugin] loading: load-as-stringvalue.lua') -local function loadRepr() - local defaultSettings = { - pretty = false; - robloxFullName = false; - robloxProperFullName = true; - robloxClassName = true; - tabs = false; - semicolons = false; - spaces = 3; - sortKeys = true; - } - - -- lua keywords - local keywords = {["and"]=true, ["break"]=true, ["do"]=true, ["else"]=true, - ["elseif"]=true, ["end"]=true, ["false"]=true, ["for"]=true, ["function"]=true, - ["if"]=true, ["in"]=true, ["local"]=true, ["nil"]=true, ["not"]=true, ["or"]=true, - ["repeat"]=true, ["return"]=true, ["then"]=true, ["true"]=true, ["until"]=true, ["while"]=true} - - local function isLuaIdentifier(str) - if type(str) ~= "string" then return false end - -- must be nonempty - if str:len() == 0 then return false end - -- can only contain a-z, A-Z, 0-9 and underscore - if str:find("[^%d%a_]") then return false end - -- cannot begin with digit - if tonumber(str:sub(1, 1)) then return false end - -- cannot be keyword - if keywords[str] then return false end - return true - end - - -- works like Instance:GetFullName(), but invalid Lua identifiers are fixed (e.g. workspace["The Dude"].Humanoid) - local function properFullName(object, usePeriod) - if object == nil or object == game then return "" end - - local s = object.Name - local usePeriod = true - if not isLuaIdentifier(s) then - s = ("[%q]"):format(s) - usePeriod = false - end - - if not object.Parent or object.Parent == game then - return s - else - return properFullName(object.Parent) .. (usePeriod and "." or "") .. s - end - end - - local depth = 0 - local shown - local INDENT - local reprSettings - - local function repr(value, reprSettings) - reprSettings = reprSettings or defaultSettings - INDENT = (" "):rep(reprSettings.spaces or defaultSettings.spaces) - if reprSettings.tabs then - INDENT = "\t" - end - - local v = value --args[1] - local tabs = INDENT:rep(depth) - - if depth == 0 then - shown = {} - end - if type(v) == "string" then - return ("%q"):format(v) - elseif type(v) == "number" then - if v == math.huge then return "math.huge" end - if v == -math.huge then return "-math.huge" end - return tonumber(v) - elseif type(v) == "boolean" then - return tostring(v) - elseif type(v) == "nil" then - return "nil" - elseif type(v) == "table" and type(v.__tostring) == "function" then - return tostring(v.__tostring(v)) - elseif type(v) == "table" and getmetatable(v) and type(getmetatable(v).__tostring) == "function" then - return tostring(getmetatable(v).__tostring(v)) - elseif type(v) == "table" then - if shown[v] then return "{CYCLIC}" end - shown[v] = true - local str = "{" .. (reprSettings.pretty and ("\n" .. INDENT .. tabs) or "") - local isArray = true - for k, v in pairs(v) do - if type(k) ~= "number" then - isArray = false - break - end - end - if isArray then - for i = 1, #v do - if i ~= 1 then - str = str .. (reprSettings.semicolons and ";" or ",") .. (reprSettings.pretty and ("\n" .. INDENT .. tabs) or " ") - end - depth = depth + 1 - str = str .. repr(v[i], reprSettings) - depth = depth - 1 - end +local function tableToString(t) + local s = '' + if type(t) == 'table' then + s = s .. '{ ' + for k, v in next, t do + if type(k) == 'number' then + s = s .. tableToString(v) else - local keyOrder = {} - local keyValueStrings = {} - for k, v in pairs(v) do - depth = depth + 1 - local kStr = isLuaIdentifier(k) and k or ("[" .. repr(k, reprSettings) .. "]") - local vStr = repr(v, reprSettings) - --[[str = str .. ("%s = %s"):format( - isLuaIdentifier(k) and k or ("[" .. repr(k, reprSettings) .. "]"), - repr(v, reprSettings) - )]] - table.insert(keyOrder, kStr) - keyValueStrings[kStr] = vStr - depth = depth - 1 - end - if reprSettings.sortKeys then table.sort(keyOrder) end - local first = true - for _, kStr in pairs(keyOrder) do - if not first then - str = str .. (reprSettings.semicolons and ";" or ",") .. (reprSettings.pretty and ("\n" .. INDENT .. tabs) or " ") - end - str = str .. ("%s = %s"):format(kStr, keyValueStrings[kStr]) - first = false - end - end - shown[v] = false - if reprSettings.pretty then - str = str .. "\n" .. tabs - end - str = str .. "}" - return str - elseif typeof then - -- Check Roblox types - if typeof(v) == "Instance" then - return (reprSettings.robloxFullName - and (reprSettings.robloxProperFullName and properFullName(v) or v:GetFullName()) - or v.Name) .. (reprSettings.robloxClassName and ((" (%s)"):format(v.ClassName)) or "") - elseif typeof(v) == "Axes" then - local s = {} - if v.X then table.insert(s, repr(Enum.Axis.X, reprSettings)) end - if v.Y then table.insert(s, repr(Enum.Axis.Y, reprSettings)) end - if v.Z then table.insert(s, repr(Enum.Axis.Z, reprSettings)) end - return ("Axes.new(%s)"):format(table.concat(s, ", ")) - elseif typeof(v) == "BrickColor" then - return ("BrickColor.new(%q)"):format(v.Name) - elseif typeof(v) == "CFrame" then - return ("CFrame.new(%s)"):format(table.concat({v:GetComponents()}, ", ")) - elseif typeof(v) == "Color3" then - return ("Color3.new(%d, %d, %d)"):format(v.r, v.g, v.b) - elseif typeof(v) == "ColorSequence" then - if #v.Keypoints > 2 then - return ("ColorSequence.new(%s)"):format(repr(v.Keypoints, reprSettings)) - else - if v.Keypoints[1].Value == v.Keypoints[2].Value then - return ("ColorSequence.new(%s)"):format(repr(v.Keypoints[1].Value, reprSettings)) - else - return ("ColorSequence.new(%s, %s)"):format( - repr(v.Keypoints[1].Value, reprSettings), - repr(v.Keypoints[2].Value, reprSettings) - ) - end - end - elseif typeof(v) == "ColorSequenceKeypoint" then - return ("ColorSequenceKeypoint.new(%d, %s)"):format(v.Time, repr(v.Value, reprSettings)) - elseif typeof(v) == "DockWidgetPluginGuiInfo" then - return ("DockWidgetPluginGuiInfo.new(%s, %s, %s, %s, %s, %s, %s)"):format( - repr(v.InitialDockState, reprSettings), - repr(v.InitialEnabled, reprSettings), - repr(v.InitialEnabledShouldOverrideRestore, reprSettings), - repr(v.FloatingXSize, reprSettings), - repr(v.FloatingYSize, reprSettings), - repr(v.MinWidth, reprSettings), - repr(v.MinHeight, reprSettings) - ) - elseif typeof(v) == "Enums" then - return "Enums" - elseif typeof(v) == "Enum" then - return ("Enum.%s"):format(tostring(v)) - elseif typeof(v) == "EnumItem" then - return ("Enum.%s.%s"):format(tostring(v.EnumType), v.Name) - elseif typeof(v) == "Faces" then - local s = {} - for _, enumItem in pairs(Enum.NormalId:GetEnumItems()) do - if v[enumItem.Name] then - table.insert(s, repr(enumItem, reprSettings)) - end - end - return ("Faces.new(%s)"):format(table.concat(s, ", ")) - elseif typeof(v) == "NumberRange" then - if v.Min == v.Max then - return ("NumberRange.new(%d)"):format(v.Min) - else - return ("NumberRange.new(%d, %d)"):format(v.Min, v.Max) - end - elseif typeof(v) == "NumberSequence" then - if #v.Keypoints > 2 then - return ("NumberSequence.new(%s)"):format(repr(v.Keypoints, reprSettings)) - else - if v.Keypoints[1].Value == v.Keypoints[2].Value then - return ("NumberSequence.new(%d)"):format(v.Keypoints[1].Value) - else - return ("NumberSequence.new(%d, %d)"):format(v.Keypoints[1].Value, v.Keypoints[2].Value) - end - end - elseif typeof(v) == "NumberSequenceKeypoint" then - if v.Envelope ~= 0 then - return ("NumberSequenceKeypoint.new(%d, %d, %d)"):format(v.Time, v.Value, v.Envelope) - else - return ("NumberSequenceKeypoint.new(%d, %d)"):format(v.Time, v.Value) - end - elseif typeof(v) == "PathWaypoint" then - return ("PathWaypoint.new(%s, %s)"):format( - repr(v.Position, reprSettings), - repr(v.Action, reprSettings) - ) - elseif typeof(v) == "PhysicalProperties" then - return ("PhysicalProperties.new(%d, %d, %d, %d, %d)"):format( - v.Density, v.Friction, v.Elasticity, v.FrictionWeight, v.ElasticityWeight - ) - elseif typeof(v) == "Random" then - return "" - elseif typeof(v) == "Ray" then - return ("Ray.new(%s, %s)"):format( - repr(v.Origin, reprSettings), - repr(v.Direction, reprSettings) - ) - elseif typeof(v) == "RBXScriptConnection" then - return "" - elseif typeof(v) == "RBXScriptSignal" then - return "" - elseif typeof(v) == "Rect" then - return ("Rect.new(%d, %d, %d, %d)"):format( - v.Min.X, v.Min.Y, v.Max.X, v.Max.Y - ) - elseif typeof(v) == "Region3" then - local min = v.CFrame.p + v.Size * -.5 - local max = v.CFrame.p + v.Size * .5 - return ("Region3.new(%s, %s)"):format( - repr(min, reprSettings), - repr(max, reprSettings) - ) - elseif typeof(v) == "Region3int16" then - return ("Region3int16.new(%s, %s)"):format( - repr(v.Min, reprSettings), - repr(v.Max, reprSettings) - ) - elseif typeof(v) == "TweenInfo" then - return ("TweenInfo.new(%d, %s, %s, %d, %s, %d)"):format( - v.Time, repr(v.EasingStyle, reprSettings), repr(v.EasingDirection, reprSettings), - v.RepeatCount, repr(v.Reverses, reprSettings), v.DelayTime - ) - elseif typeof(v) == "UDim" then - return ("UDim.new(%d, %d)"):format( - v.Scale, v.Offset - ) - elseif typeof(v) == "UDim2" then - return ("UDim2.new(%d, %d, %d, %d)"):format( - v.X.Scale, v.X.Offset, v.Y.Scale, v.Y.Offset - ) - elseif typeof(v) == "Vector2" then - return ("Vector2.new(%d, %d)"):format(v.X, v.Y) - elseif typeof(v) == "Vector2int16" then - return ("Vector2int16.new(%d, %d)"):format(v.X, v.Y) - elseif typeof(v) == "Vector3" then - return ("Vector3.new(%d, %d, %d)"):format(v.X, v.Y, v.Z) - elseif typeof(v) == "Vector3int16" then - return ("Vector3int16.new(%d, %d, %d)"):format(v.X, v.Y, v.Z) - elseif typeof(v) == "DateTime" then - return ("DateTime.fromIsoDate(%q)"):format(v:ToIsoDate()) - else - return "" + s = s .. k .. ' = ' .. tableToString(v) end - else - return "<" .. type(v) .. ">" end + s = s .. ' }' + elseif type(t) == 'string' then + s = s .. '"' .. t .. '"' + else + s = s .. tostring(t) end - - return repr + return s end -local repr = loadRepr() - return function(options) - print(('[plugin] create with: %s'):format(repr(options))) + print(('[plugin] create with: %s'):format(tableToString(options))) options.extensions = options.extensions or {} return { diff --git a/test-projects/plugins/src/hello.md b/test-projects/plugins/src/Hello.md similarity index 60% rename from test-projects/plugins/src/hello.md rename to test-projects/plugins/src/Hello.md index 234674249..f01722bb1 100644 --- a/test-projects/plugins/src/hello.md +++ b/test-projects/plugins/src/Hello.md @@ -1,3 +1,3 @@ # Markdown -Woo! +WooHoo! From 17a1db6f6a0a5a2babaff4144a8a92606af85124 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 15:08:24 -0500 Subject: [PATCH 15/22] read files using plugin library --- PLUGIN-DESIGN.md | 82 +++++++++---------- src/load_file.rs | 10 +-- src/plugin_env.rs | 52 ++++++++---- src/serve_session.rs | 5 +- test-projects/plugins/load-as-stringvalue.lua | 3 +- 5 files changed, 80 insertions(+), 72 deletions(-) diff --git a/PLUGIN-DESIGN.md b/PLUGIN-DESIGN.md index 027d47de8..2df863f4d 100644 --- a/PLUGIN-DESIGN.md +++ b/PLUGIN-DESIGN.md @@ -8,13 +8,13 @@ transformation. As discussed in [#55](https://github.com/rojo-rbx/rojo/issues/55) and as initially explored in [#257](https://github.com/rojo-rbx/rojo/pull/257), plugins as Lua scripts seem to be a good starting -point. This is quite similar to the way Rollup.js plugins work, although they are implemented with -JS. Rollup.js is a bundler which actually performs a similar job to Rojo in the JS world by taking a -number of source files and converting them into a single output bundle. - -This proposal takes strong influence from the [Rollup.js plugins -API](https://rollupjs.org/guide/en/#plugins-overview) which is a joy to use for both plugin -developers and end-users. +point. This concept reminded me of Rollup.js plugins. [Rollup](https://rollupjs.org/guide/en/) is a +JS bundler which performs a similar job to Rojo in the JS world by taking a number of source files +and producing a single output bundle. [Rollup +plugins](https://rollupjs.org/guide/en/#plugins-overview) are written as JS files, and passed to the +primary Rollup config file. Rollup then calls in to plugin "hooks" at special times during the +bundling process. I have found Rollup plugins to be an excellent interface for both plugin +developers and end-users and therefore I have based this proposal on their API. ## Project file changes @@ -49,13 +49,7 @@ field set to its value, and `options` set to its default. "github.com/owner/remote-plugin-from-head", { "source": "plugin-with-options.lua", "options": { "some": "option" } } ], - "tree": { - "$className": "DataModel", - "ServerScriptService": { - "$className": "ServerScriptService", - "$path": "src" - } - } + ... } ``` @@ -76,7 +70,7 @@ type PluginInstance = { load?: (id: string) -> string, } --- TODO: Define properly. For now, this is basically just the JSON converted to Lua +-- TODO: Define properly. This is basically just the JSON converted to Lua type ProjectDescription = { ... } type CreatePluginFunction = (options: {[string]: any}) -> PluginInstance @@ -85,28 +79,6 @@ type CreatePluginFunction = (options: {[string]: any}) -> PluginInstance In this way, plugins have the opportunity to customize their hooks based on the options provided by the user in the project file. -## Plugin environment - -The plugin environment is created in the following way: - -1. Create a new Lua context. -1. Add a global `rojo` table which is the entry point to the [Plugin library](#plugin-library) -1. Initialize an empty `_G.plugins` table. -1. For each plugin description in the project file: - 1. Convert the plugin options from the project file from JSON to a Lua table. - 1. If the `source` field is a GitHub URL, download the plugin directory from the repo with the - specified version tag (if no tag, from the head of the default branch) into a local - `.rojo-plugins` directory with the repo identifier as its name. It is recommended that users - add `.rojo-plugins` to their `.gitignore` file. The root of the plugin will be called - `main.lua`. - 1. Load and evaluate the file contents into the Lua context to get a handle to the - `CreatePluginFunction` - 1. Call the `CreatePluginFunction` with the converted options to get a handle of the result. - 1. Push the result at the end of the `_G.plugins` table - -If at any point there is an error in the above steps, Rojo should quit with an appropriate error -message. - ## Plugin instance - `name` @@ -129,13 +101,35 @@ message. - **Optional**: Takes a file path and returns the file contents that should be interpreted by Rojo. The first plugin to return a non-nil value per id wins. +## Plugin environment + +The plugin environment is created in the following way: + +1. Create a new Lua context. +1. Add a global `rojo` table which is the entry point to the [Plugin library](#plugin-library) +1. Initialize an empty `_G.plugins` table. +1. For each plugin description in the project file: + 1. Convert the plugin options from the project file from JSON to a Lua table. + 1. If the `source` field is a GitHub URL, download the plugin directory from the repo with the + specified version tag (if no tag, from the head of the default branch) into a local + `.rojo-plugins` directory with the repo identifier as its name. It is recommended that users + add `.rojo-plugins` to their `.gitignore` file. The root of the plugin will be called + `main.lua`. + 1. Load and evaluate the file contents into the Lua context to get a handle to the + `CreatePluginFunction` + 1. Call the `CreatePluginFunction` with the converted options to get a handle of the result. + 1. Push the result at the end of the `_G.plugins` table + +If at any point there is an error in the above steps, Rojo should quit with an appropriate error +message. + ## Plugin library Accessible via the `rojo` global, the plugin library offers helper methods for plugins: - `toJson(value: any) -> string` - Converts a Lua value to a JSON string. -- `readFile(id: string) -> string` +- `readFileAsUtf8(id: string) -> string` - Reads the contents of a file from a file path using the VFS. Plugins should always read files via this method rather than directly from the file system. - `getExtension(id: string) -> boolean` @@ -164,7 +158,7 @@ return function(options) name = "moonscript", load = function(id) if rojo.hasExtension(id, 'lua') then - local contents = rojo.readFile(id) + local contents = rojo.readFileAsUtf8(id) local tree, err = parse.string(contents) assert(tree, err) @@ -194,7 +188,7 @@ return function(options) name = "minifier", load = function(id) if rojo.hasExtension(id, 'lua') then - local contents = rojo.readFile(id) + local contents = rojo.readFileAsUtf8(id) return minifier(contents) end end @@ -217,7 +211,7 @@ return function(options) end, load = function(id) if rojo.hasExtension(id, 'md') then - local contents = rojo.readFile(id) + local contents = rojo.readFileAsUtf8(id) local frontmatter = parseFrontmatter(contents) local className = frontmatter.className or 'StringValue' @@ -262,7 +256,7 @@ return function(options) local idExt = rojo.getExtension(id) for _, ext in next, options.extensions do if ext == idExt then - local contents = rojo.readFile(id) + local contents = rojo.readFileAsUtf8(id) local jsonEncoded = contents:gsub('\r', '\\r'):gsub('\n', '\\n') return rojo.toJson({ @@ -306,7 +300,7 @@ return function(options) end, load = function(id) if rojo.hasExtension(id, 'lua') then - local contents = rojo.readFile(id) + local contents = rojo.readFileAsUtf8(id) -- This function will look for require 'file/path' statements in the source and replace -- them with Roblox require(instance.path) statements based on the project's configuration @@ -356,7 +350,7 @@ This proposal could be split up into milestones without all the features being p to simplify development. Here is a proposal for the order to implement each milestone: 1. Loading plugins from local paths -1. Minimum Rojo plugin library (`readFile`) +1. Minimum Rojo plugin library (`readFileAsUtf8`) 1. Calling hooks at the appropriate time 1. Start with `middleware`, `load` 1. Add `syncStart`, `syncEnd`, `resolve` diff --git a/src/load_file.rs b/src/load_file.rs index 327ec5573..53c2342a4 100644 --- a/src/load_file.rs +++ b/src/load_file.rs @@ -1,6 +1,5 @@ -use anyhow::Context; use memofs::Vfs; -use std::{path::Path, str, sync::Arc}; +use std::{path::Path, sync::Arc}; use crate::plugin_env::PluginEnv; @@ -9,16 +8,13 @@ pub fn load_file( plugin_env: &PluginEnv, path: &Path, ) -> Result>, anyhow::Error> { - let contents = vfs.read(path)?; - let contents_str = str::from_utf8(&contents) - .with_context(|| format!("File was not valid UTF-8: {}", path.display()))?; - - let plugin_result = plugin_env.load(path.to_str().unwrap(), contents_str); + let plugin_result = plugin_env.load(path.to_str().unwrap()); match plugin_result { Ok(Some(data)) => return Ok(Arc::new(data.as_bytes().to_vec())), Ok(None) => {} Err(_) => {} } + let contents = vfs.read(path)?; return Ok(contents); } diff --git a/src/plugin_env.rs b/src/plugin_env.rs index 47f5a1ced..eca7c173f 100644 --- a/src/plugin_env.rs +++ b/src/plugin_env.rs @@ -1,16 +1,18 @@ +use memofs::Vfs; use rlua::{Function, Lua, Table}; -use std::{fs, str::FromStr}; +use std::{fs, path::Path, str, str::FromStr, sync::Arc}; use crate::snapshot_middleware::SnapshotMiddleware; pub struct PluginEnv { lua: Lua, + vfs: Arc, } impl PluginEnv { - pub fn new() -> Self { + pub fn new(vfs: Arc) -> Self { let lua = Lua::new(); - PluginEnv { lua } + PluginEnv { lua, vfs } } pub fn init(&self) -> Result<(), rlua::Error> { @@ -20,17 +22,8 @@ impl PluginEnv { let plugins_table = lua_ctx.create_table()?; globals.set("plugins", plugins_table)?; - let run_plugins_fn = lua_ctx.create_function(|lua_ctx, id: String| { - let plugins: Table = lua_ctx.globals().get("plugins")?; - let id_ref: &str = &id; - for plugin in plugins.sequence_values::
() { - let load: Function = plugin?.get("load")?; - load.call(id_ref)?; - } - - Ok(()) - })?; - globals.set("runPlugins", run_plugins_fn)?; + let plugin_library_table = lua_ctx.create_table()?; + globals.set("rojo", plugin_library_table)?; Ok::<(), rlua::Error>(()) }) @@ -112,8 +105,31 @@ impl PluginEnv { }) } - pub fn middleware(&self, id: &str) -> Result, rlua::Error> { + pub fn context_with_vfs(&self, f: F) -> Result + where + F: FnOnce(rlua::Context) -> Result, + { + let vfs = Arc::clone(&self.vfs); + self.lua.context(|lua_ctx| { + lua_ctx.scope(|scope| { + let globals = lua_ctx.globals(); + let plugin_library_table: Table = globals.get("rojo")?; + let read_file_fn = scope.create_function_mut(|_, id: String| { + let path = Path::new(&id); + let contents = vfs.read(path).unwrap(); + let contents_str = str::from_utf8(&contents).unwrap(); + Ok::(contents_str.to_owned()) + })?; + plugin_library_table.set("readFileAsUtf8", read_file_fn)?; + + f(lua_ctx) + }) + }) + } + + pub fn middleware(&self, id: &str) -> Result, rlua::Error> { + self.context_with_vfs(|lua_ctx| { let globals = lua_ctx.globals(); let plugins: Table = globals.get("plugins")?; @@ -133,14 +149,14 @@ impl PluginEnv { }) } - pub fn load(&self, id: &str, contents: &str) -> Result, rlua::Error> { - self.lua.context(|lua_ctx| { + pub fn load(&self, id: &str) -> Result, rlua::Error> { + self.context_with_vfs(|lua_ctx| { let globals = lua_ctx.globals(); let plugins: Table = globals.get("plugins")?; for plugin in plugins.sequence_values::
() { let load_fn: Function = plugin?.get("load")?; - let load_str: Option = load_fn.call((id, contents))?; + let load_str: Option = load_fn.call(id)?; if load_str.is_some() { return Ok(load_str); } diff --git a/src/serve_session.rs b/src/serve_session.rs index a83cc885f..15ad4ed48 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -144,7 +144,9 @@ impl ServeSession { } }; - let plugin_env = PluginEnv::new(); + let vfs = Arc::new(vfs); + + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); match plugin_env.init() { Ok(_) => (), Err(e) => return Err(ServeSessionError::Plugin { source: e }), @@ -186,7 +188,6 @@ impl ServeSession { let tree = Arc::new(Mutex::new(tree)); let message_queue = Arc::new(message_queue); - let vfs = Arc::new(vfs); let plugin_env = Arc::new(Mutex::new(plugin_env)); let (tree_mutation_sender, tree_mutation_receiver) = crossbeam_channel::unbounded(); diff --git a/test-projects/plugins/load-as-stringvalue.lua b/test-projects/plugins/load-as-stringvalue.lua index 2c270b463..71d4ab16f 100644 --- a/test-projects/plugins/load-as-stringvalue.lua +++ b/test-projects/plugins/load-as-stringvalue.lua @@ -37,11 +37,12 @@ return function(options) end print('[plugin] skipping') end, - load = function(id, contents) + load = function(id) print(('[plugin] load: %s'):format(id)) local idExt = id:match('%.(%w+)$') for _, ext in next, options.extensions do if ext == idExt then + local contents = rojo.readFileAsUtf8(id) print(('[plugin] matched: %s'):format(ext)) local encoded = contents:gsub('\n', '\\n') return ('{"ClassName": "StringValue", "Properties": { "Value": "%s" }}'):format(encoded) From cb18cfe48905b74fc51855c8bcc8560621c882f6 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 15:21:17 -0500 Subject: [PATCH 16/22] use context_with_cfs when loading plugins, restrict lua std libraries in plugins --- src/plugin_env.rs | 67 ++++++++++++++++------------ test-projects/plugins/src/Another.md | 1 + 2 files changed, 40 insertions(+), 28 deletions(-) create mode 100644 test-projects/plugins/src/Another.md diff --git a/src/plugin_env.rs b/src/plugin_env.rs index eca7c173f..8d2a977c8 100644 --- a/src/plugin_env.rs +++ b/src/plugin_env.rs @@ -1,5 +1,5 @@ use memofs::Vfs; -use rlua::{Function, Lua, Table}; +use rlua::{Function, Lua, StdLib, Table}; use std::{fs, path::Path, str, str::FromStr, sync::Arc}; use crate::snapshot_middleware::SnapshotMiddleware; @@ -11,7 +11,14 @@ pub struct PluginEnv { impl PluginEnv { pub fn new(vfs: Arc) -> Self { - let lua = Lua::new(); + let lua = Lua::new_with( + StdLib::BASE + | StdLib::TABLE + | StdLib::STRING + | StdLib::UTF8 + | StdLib::MATH + | StdLib::PACKAGE, + ); PluginEnv { lua, vfs } } @@ -29,6 +36,33 @@ impl PluginEnv { }) } + pub fn context_with_vfs(&self, f: F) -> Result + where + F: FnOnce(rlua::Context) -> Result, + { + // We cannot just create a global function that has access to the vfs and call it whenever + // we want because that would be unsafe as Lua is unaware of the lifetime of vfs. Therefore + // we have to create a limited lifetime scope that has access to the vfs and define the + // function each time plugin code is executed. + let vfs = Arc::clone(&self.vfs); + + self.lua.context(|lua_ctx| { + lua_ctx.scope(|scope| { + let globals = lua_ctx.globals(); + let plugin_library_table: Table = globals.get("rojo")?; + let read_file_fn = scope.create_function_mut(|_, id: String| { + let path = Path::new(&id); + let contents = vfs.read(path).unwrap(); + let contents_str = str::from_utf8(&contents).unwrap(); + Ok::(contents_str.to_owned()) + })?; + plugin_library_table.set("readFileAsUtf8", read_file_fn)?; + + f(lua_ctx) + }) + }) + } + fn load_plugin_source(&self, plugin_source: &str) -> String { // TODO: Support downloading and caching plugins fs::read_to_string(plugin_source).unwrap() @@ -41,7 +75,7 @@ impl PluginEnv { ) -> Result<(), rlua::Error> { let plugin_lua = &(self.load_plugin_source(plugin_source)); - self.lua.context(|lua_ctx| { + self.context_with_vfs(|lua_ctx| { let globals = lua_ctx.globals(); let create_plugin_fn: Option = @@ -88,8 +122,8 @@ impl PluginEnv { ) .to_string(), )) - }, - v => v + } + v => v, }; log::trace!( @@ -105,29 +139,6 @@ impl PluginEnv { }) } - pub fn context_with_vfs(&self, f: F) -> Result - where - F: FnOnce(rlua::Context) -> Result, - { - let vfs = Arc::clone(&self.vfs); - - self.lua.context(|lua_ctx| { - lua_ctx.scope(|scope| { - let globals = lua_ctx.globals(); - let plugin_library_table: Table = globals.get("rojo")?; - let read_file_fn = scope.create_function_mut(|_, id: String| { - let path = Path::new(&id); - let contents = vfs.read(path).unwrap(); - let contents_str = str::from_utf8(&contents).unwrap(); - Ok::(contents_str.to_owned()) - })?; - plugin_library_table.set("readFileAsUtf8", read_file_fn)?; - - f(lua_ctx) - }) - }) - } - pub fn middleware(&self, id: &str) -> Result, rlua::Error> { self.context_with_vfs(|lua_ctx| { let globals = lua_ctx.globals(); diff --git a/test-projects/plugins/src/Another.md b/test-projects/plugins/src/Another.md new file mode 100644 index 000000000..56323ef55 --- /dev/null +++ b/test-projects/plugins/src/Another.md @@ -0,0 +1 @@ +Some **bold** text. From b1081cd4dde341ddc265a4069d882d6a3b73e0d6 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 17:07:44 -0500 Subject: [PATCH 17/22] finish porting all middlewares --- PLUGIN-DESIGN.md | 7 +- src/plugin_env.rs | 12 +- src/serve_session.rs | 3 - src/snapshot_middleware/csv.rs | 53 +++++-- src/snapshot_middleware/dir.rs | 10 +- src/snapshot_middleware/json.rs | 18 ++- src/snapshot_middleware/json_model.rs | 14 +- src/snapshot_middleware/lua.rs | 124 ++++++++++----- src/snapshot_middleware/mod.rs | 150 ++++++++++++++---- src/snapshot_middleware/project.rs | 42 ++--- src/snapshot_middleware/rbxm.rs | 24 ++- src/snapshot_middleware/rbxmx.rs | 24 ++- src/snapshot_middleware/txt.rs | 34 ++-- src/snapshot_middleware/util.rs | 14 +- test-projects/plugins/default.project.json | 1 + test-projects/plugins/fake-moonscript.lua | 39 +++++ test-projects/plugins/load-as-stringvalue.lua | 16 +- test-projects/plugins/src/Another.md | 1 - test-projects/plugins/src/Document.md | 3 + test-projects/plugins/src/Hello.client.moon | 1 + test-projects/plugins/src/Hello.md | 3 - test-projects/plugins/src/Hello.moon | 1 + test-projects/plugins/src/Hello.server.moon | 1 + 23 files changed, 435 insertions(+), 160 deletions(-) create mode 100644 test-projects/plugins/fake-moonscript.lua delete mode 100644 test-projects/plugins/src/Another.md create mode 100644 test-projects/plugins/src/Document.md create mode 100644 test-projects/plugins/src/Hello.client.moon delete mode 100644 test-projects/plugins/src/Hello.md create mode 100644 test-projects/plugins/src/Hello.moon create mode 100644 test-projects/plugins/src/Hello.server.moon diff --git a/PLUGIN-DESIGN.md b/PLUGIN-DESIGN.md index 2df863f4d..ffcc6189d 100644 --- a/PLUGIN-DESIGN.md +++ b/PLUGIN-DESIGN.md @@ -156,8 +156,13 @@ local compile = require 'moonscript.compile' return function(options) return { name = "moonscript", + middleware = function(id) + if rojo.hasExtension(id, 'moon') then + return 'lua' + end + end, load = function(id) - if rojo.hasExtension(id, 'lua') then + if rojo.hasExtension(id, 'moon') then local contents = rojo.readFileAsUtf8(id) local tree, err = parse.string(contents) diff --git a/src/plugin_env.rs b/src/plugin_env.rs index 8d2a977c8..18f1b6656 100644 --- a/src/plugin_env.rs +++ b/src/plugin_env.rs @@ -139,24 +139,28 @@ impl PluginEnv { }) } - pub fn middleware(&self, id: &str) -> Result, rlua::Error> { + pub fn middleware( + &self, + id: &str, + ) -> Result<(Option, Option), rlua::Error> { self.context_with_vfs(|lua_ctx| { let globals = lua_ctx.globals(); let plugins: Table = globals.get("plugins")?; for plugin in plugins.sequence_values::
() { let middleware_fn: Function = plugin?.get("middleware")?; - let middleware_str: Option = middleware_fn.call(id)?; + let (middleware_str, name): (Option, Option) = + middleware_fn.call(id)?; let middleware_enum = match middleware_str { Some(str) => SnapshotMiddleware::from_str(&str).ok(), None => None, }; if middleware_enum.is_some() { - return Ok(middleware_enum); + return Ok((middleware_enum, name)); } } - Ok(None) + Ok((None, None)) }) } diff --git a/src/serve_session.rs b/src/serve_session.rs index 15ad4ed48..a5a4d6e34 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -109,8 +109,6 @@ pub struct ServeSession { /// A channel to send mutation requests on. These will be handled by the /// ChangeProcessor and trigger changes in the tree. tree_mutation_sender: Sender, - - plugin_env: Arc>, } impl ServeSession { @@ -210,7 +208,6 @@ impl ServeSession { message_queue, tree_mutation_sender, vfs, - plugin_env, }) } diff --git a/src/snapshot_middleware/csv.rs b/src/snapshot_middleware/csv.rs index a8f271808..b6e282897 100644 --- a/src/snapshot_middleware/csv.rs +++ b/src/snapshot_middleware/csv.rs @@ -5,19 +5,24 @@ use maplit::hashmap; use memofs::{IoResultExt, Vfs}; use serde::Serialize; -use crate::snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}; +use crate::{ + load_file::load_file, + plugin_env::PluginEnv, + snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, +}; -use super::{meta_file::AdjacentMetadata, util::PathExt}; +use super::meta_file::AdjacentMetadata; pub fn snapshot_csv( _context: &InstanceContext, vfs: &Vfs, + plugin_env: &PluginEnv, path: &Path, + name: &str, ) -> anyhow::Result> { - let name = path.file_name_trim_end(".csv")?; - + // TODO: This is probably broken + let contents = load_file(vfs, plugin_env, path)?; let meta_path = path.with_file_name(format!("{}.meta.json", name)); - let contents = vfs.read(path)?; let table_contents = convert_localization_csv(&contents).with_context(|| { format!( @@ -125,6 +130,8 @@ fn convert_localization_csv(contents: &[u8]) -> Result { #[cfg(test)] mod test { + use std::sync::Arc; + use super::*; use memofs::{InMemoryFs, VfsSnapshot}; @@ -142,12 +149,20 @@ Ack,Ack!,,An exclamation of despair,¡Ay!"#, ) .unwrap(); - let mut vfs = Vfs::new(imfs); + let mut vfs = Arc::new(Vfs::new(imfs)); - let instance_snapshot = - snapshot_csv(&InstanceContext::default(), &mut vfs, Path::new("/foo.csv")) - .unwrap() - .unwrap(); + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); + plugin_env.init().unwrap(); + + let instance_snapshot = snapshot_csv( + &InstanceContext::default(), + &mut vfs, + &plugin_env, + Path::new("/foo.csv"), + "foo", + ) + .unwrap() + .unwrap(); insta::assert_yaml_snapshot!(instance_snapshot); } @@ -170,12 +185,20 @@ Ack,Ack!,,An exclamation of despair,¡Ay!"#, ) .unwrap(); - let mut vfs = Vfs::new(imfs); + let mut vfs = Arc::new(Vfs::new(imfs)); + + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); + plugin_env.init().unwrap(); - let instance_snapshot = - snapshot_csv(&InstanceContext::default(), &mut vfs, Path::new("/foo.csv")) - .unwrap() - .unwrap(); + let instance_snapshot = snapshot_csv( + &InstanceContext::default(), + &mut vfs, + &plugin_env, + Path::new("/foo.csv"), + "foo", + ) + .unwrap() + .unwrap(); insta::assert_yaml_snapshot!(instance_snapshot); } diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index c3514353f..f85560d7e 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -77,6 +77,8 @@ pub fn snapshot_dir( #[cfg(test)] mod test { + use std::sync::Arc; + use super::*; use maplit::hashmap; @@ -88,9 +90,9 @@ mod test { imfs.load_snapshot("/foo", VfsSnapshot::empty_dir()) .unwrap(); - let mut vfs = Vfs::new(imfs); + let mut vfs = Arc::new(Vfs::new(imfs)); - let plugin_env = PluginEnv::new(); + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); plugin_env.init().unwrap(); let instance_snapshot = snapshot_dir( @@ -116,9 +118,9 @@ mod test { ) .unwrap(); - let mut vfs = Vfs::new(imfs); + let mut vfs = Arc::new(Vfs::new(imfs)); - let plugin_env = PluginEnv::new(); + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); plugin_env.init().unwrap(); let instance_snapshot = snapshot_dir( diff --git a/src/snapshot_middleware/json.rs b/src/snapshot_middleware/json.rs index 8c7f369e3..7e47cf208 100644 --- a/src/snapshot_middleware/json.rs +++ b/src/snapshot_middleware/json.rs @@ -5,19 +5,22 @@ use maplit::hashmap; use memofs::{IoResultExt, Vfs}; use crate::{ + load_file::load_file, lua_ast::{Expression, Statement}, + plugin_env::PluginEnv, snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, }; -use super::{meta_file::AdjacentMetadata, util::PathExt}; +use super::meta_file::AdjacentMetadata; pub fn snapshot_json( context: &InstanceContext, vfs: &Vfs, + plugin_env: &PluginEnv, path: &Path, + name: &str, ) -> anyhow::Result> { - let name = path.file_name_trim_end(".json")?; - let contents = vfs.read(path)?; + let contents = load_file(vfs, plugin_env, path)?; let value: serde_json::Value = serde_json::from_slice(&contents) .with_context(|| format!("File contains malformed JSON: {}", path.display()))?; @@ -75,6 +78,8 @@ fn json_to_lua_value(value: serde_json::Value) -> Expression { #[cfg(test)] mod test { + use std::sync::Arc; + use super::*; use memofs::{InMemoryFs, VfsSnapshot}; @@ -101,12 +106,17 @@ mod test { ) .unwrap(); - let mut vfs = Vfs::new(imfs.clone()); + let mut vfs = Arc::new(Vfs::new(imfs.clone())); + + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); + plugin_env.init().unwrap(); let instance_snapshot = snapshot_json( &InstanceContext::default(), &mut vfs, + &plugin_env, Path::new("/foo.json"), + "foo", ) .unwrap() .unwrap(); diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 97347d8b0..d8694c9b6 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -11,19 +11,14 @@ use crate::{ snapshot::{InstanceContext, InstanceSnapshot}, }; -use super::util::PathExt; - pub fn snapshot_json_model( context: &InstanceContext, vfs: &Vfs, plugin_env: &PluginEnv, path: &Path, + name: &str, ) -> anyhow::Result> { - // let name = path.file_name_trim_end(".model.json")?; - let name = path.file_name().and_then(|s| s.to_str()).unwrap(); - let contents = load_file(vfs, plugin_env, path)?; - // let contents = vfs.read(path)?; let contents_str = str::from_utf8(&contents) .with_context(|| format!("File was not valid UTF-8: {}", path.display()))?; @@ -106,6 +101,8 @@ impl JsonModelCore { #[cfg(test)] mod test { + use std::sync::Arc; + use super::*; use memofs::{InMemoryFs, VfsSnapshot}; @@ -135,9 +132,9 @@ mod test { ) .unwrap(); - let mut vfs = Vfs::new(imfs); + let mut vfs = Arc::new(Vfs::new(imfs)); - let plugin_env = PluginEnv::new(); + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); plugin_env.init().unwrap(); let instance_snapshot = snapshot_json_model( @@ -145,6 +142,7 @@ mod test { &mut vfs, &plugin_env, Path::new("/foo.model.json"), + "foo", ) .unwrap() .unwrap(); diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index d21a549c1..ef48adb1f 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -5,40 +5,32 @@ use maplit::hashmap; use memofs::{IoResultExt, Vfs}; use crate::{ + load_file::load_file, plugin_env::PluginEnv, snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, }; -use super::{dir::snapshot_dir, meta_file::AdjacentMetadata, util::match_trailing}; +use super::{dir::snapshot_dir, meta_file::AdjacentMetadata}; /// Core routine for turning Lua files into snapshots. pub fn snapshot_lua( context: &InstanceContext, vfs: &Vfs, + plugin_env: &PluginEnv, path: &Path, + name: &str, + class_name: &str, ) -> anyhow::Result> { - let file_name = path.file_name().unwrap().to_string_lossy(); - - let (class_name, instance_name) = if let Some(name) = match_trailing(&file_name, ".server.lua") - { - ("Script", name) - } else if let Some(name) = match_trailing(&file_name, ".client.lua") { - ("LocalScript", name) - } else if let Some(name) = match_trailing(&file_name, ".lua") { - ("ModuleScript", name) - } else { - return Ok(None); - }; - - let contents = vfs.read(path)?; + let contents = load_file(vfs, plugin_env, path)?; let contents_str = str::from_utf8(&contents) .with_context(|| format!("File was not valid UTF-8: {}", path.display()))? .to_owned(); - let meta_path = path.with_file_name(format!("{}.meta.json", instance_name)); + // TODO: I think this is broken + let meta_path = path.with_file_name(format!("{}.meta.json", name)); let mut snapshot = InstanceSnapshot::new() - .name(instance_name) + .name(name) .class_name(class_name) .properties(hashmap! { "Source".to_owned() => contents_str.into(), @@ -68,6 +60,8 @@ pub fn snapshot_lua_init( vfs: &Vfs, plugin_env: &PluginEnv, init_path: &Path, + name: &str, + class_name: &str, ) -> anyhow::Result> { let folder_path = init_path.parent().unwrap(); let dir_snapshot = snapshot_dir(context, vfs, plugin_env, folder_path)?.unwrap(); @@ -84,9 +78,10 @@ pub fn snapshot_lua_init( ); } - let mut init_snapshot = snapshot_lua(context, vfs, init_path)?.unwrap(); + let mut init_snapshot = + snapshot_lua(context, vfs, plugin_env, init_path, name, class_name)?.unwrap(); - init_snapshot.name = dir_snapshot.name; + // init_snapshot.name = dir_snapshot.name; init_snapshot.children = dir_snapshot.children; init_snapshot.metadata = dir_snapshot.metadata; @@ -95,6 +90,8 @@ pub fn snapshot_lua_init( #[cfg(test)] mod test { + use std::sync::Arc; + use super::*; use memofs::{InMemoryFs, VfsSnapshot}; @@ -105,12 +102,21 @@ mod test { imfs.load_snapshot("/foo.lua", VfsSnapshot::file("Hello there!")) .unwrap(); - let mut vfs = Vfs::new(imfs); + let mut vfs = Arc::new(Vfs::new(imfs)); - let instance_snapshot = - snapshot_lua(&InstanceContext::default(), &mut vfs, Path::new("/foo.lua")) - .unwrap() - .unwrap(); + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); + plugin_env.init().unwrap(); + + let instance_snapshot = snapshot_lua( + &InstanceContext::default(), + &mut vfs, + &plugin_env, + Path::new("/foo.lua"), + "foo", + "ModuleScript", + ) + .unwrap() + .unwrap(); insta::assert_yaml_snapshot!(instance_snapshot); } @@ -121,12 +127,18 @@ mod test { imfs.load_snapshot("/foo.server.lua", VfsSnapshot::file("Hello there!")) .unwrap(); - let mut vfs = Vfs::new(imfs); + let mut vfs = Arc::new(Vfs::new(imfs)); + + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); + plugin_env.init().unwrap(); let instance_snapshot = snapshot_lua( &InstanceContext::default(), &mut vfs, + &plugin_env, Path::new("/foo.server.lua"), + "foo", + "Script", ) .unwrap() .unwrap(); @@ -140,12 +152,18 @@ mod test { imfs.load_snapshot("/foo.client.lua", VfsSnapshot::file("Hello there!")) .unwrap(); - let mut vfs = Vfs::new(imfs); + let mut vfs = Arc::new(Vfs::new(imfs)); + + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); + plugin_env.init().unwrap(); let instance_snapshot = snapshot_lua( &InstanceContext::default(), &mut vfs, + &plugin_env, Path::new("/foo.client.lua"), + "foo", + "LocalScript", ) .unwrap() .unwrap(); @@ -165,12 +183,21 @@ mod test { ) .unwrap(); - let mut vfs = Vfs::new(imfs); + let mut vfs = Arc::new(Vfs::new(imfs)); - let instance_snapshot = - snapshot_lua(&InstanceContext::default(), &mut vfs, Path::new("/root")) - .unwrap() - .unwrap(); + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); + plugin_env.init().unwrap(); + + let instance_snapshot = snapshot_lua( + &InstanceContext::default(), + &mut vfs, + &plugin_env, + Path::new("/root"), + "root", + "ModuleScript", + ) + .unwrap() + .unwrap(); insta::assert_yaml_snapshot!(instance_snapshot); } @@ -192,12 +219,21 @@ mod test { ) .unwrap(); - let mut vfs = Vfs::new(imfs); + let mut vfs = Arc::new(Vfs::new(imfs)); - let instance_snapshot = - snapshot_lua(&InstanceContext::default(), &mut vfs, Path::new("/foo.lua")) - .unwrap() - .unwrap(); + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); + plugin_env.init().unwrap(); + + let instance_snapshot = snapshot_lua( + &InstanceContext::default(), + &mut vfs, + &plugin_env, + Path::new("/foo.lua"), + "foo", + "ModuleScript", + ) + .unwrap() + .unwrap(); insta::assert_yaml_snapshot!(instance_snapshot); } @@ -219,12 +255,18 @@ mod test { ) .unwrap(); - let mut vfs = Vfs::new(imfs); + let mut vfs = Arc::new(Vfs::new(imfs)); + + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); + plugin_env.init().unwrap(); let instance_snapshot = snapshot_lua( &InstanceContext::default(), &mut vfs, + &plugin_env, Path::new("/foo.server.lua"), + "foo", + "Script", ) .unwrap() .unwrap(); @@ -251,12 +293,18 @@ mod test { ) .unwrap(); - let mut vfs = Vfs::new(imfs); + let mut vfs = Arc::new(Vfs::new(imfs)); + + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); + plugin_env.init().unwrap(); let instance_snapshot = snapshot_lua( &InstanceContext::default(), &mut vfs, + &plugin_env, Path::new("/bar.server.lua"), + "bar", + "Script", ) .unwrap() .unwrap(); diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 76d9b81f8..c902d4332 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -47,7 +47,9 @@ pub enum SnapshotMiddleware { Dir, Json, JsonModel, - Lua, + LuaModule, + LuaClient, + LuaServer, Project, Rbxm, Rbxmx, @@ -63,7 +65,9 @@ impl FromStr for SnapshotMiddleware { "dir" => Ok(SnapshotMiddleware::Dir), "json" => Ok(SnapshotMiddleware::Json), "json_model" => Ok(SnapshotMiddleware::JsonModel), - "lua" => Ok(SnapshotMiddleware::Lua), + "lua_module" => Ok(SnapshotMiddleware::LuaModule), + "lua_server" => Ok(SnapshotMiddleware::LuaServer), + "lua_client" => Ok(SnapshotMiddleware::LuaClient), "project" => Ok(SnapshotMiddleware::Project), "rbxm" => Ok(SnapshotMiddleware::Rbxm), "rbxmx" => Ok(SnapshotMiddleware::Rbxmx), @@ -94,66 +98,158 @@ pub fn snapshot_from_vfs( let init_path = path.join("init.lua"); if vfs.metadata(&init_path).with_not_found()?.is_some() { - return snapshot_lua_init(context, vfs, plugin_env, &init_path); + return snapshot_lua_init( + context, + vfs, + plugin_env, + &init_path, + &path.file_name().unwrap().to_string_lossy(), + "ModuleScript", + ); } let init_path = path.join("init.server.lua"); if vfs.metadata(&init_path).with_not_found()?.is_some() { - return snapshot_lua_init(context, vfs, plugin_env, &init_path); + return snapshot_lua_init( + context, + vfs, + plugin_env, + &init_path, + &path.file_name().unwrap().to_string_lossy(), + "Script", + ); } let init_path = path.join("init.client.lua"); if vfs.metadata(&init_path).with_not_found()?.is_some() { - return snapshot_lua_init(context, vfs, plugin_env, &init_path); + return snapshot_lua_init( + context, + vfs, + plugin_env, + &init_path, + &path.file_name().unwrap().to_string_lossy(), + "LocalScript", + ); } snapshot_dir(context, vfs, plugin_env, path) } else { - let mut middleware = plugin_env.middleware(path.to_str().unwrap())?; + let mut middleware: (Option, Option) = + plugin_env.middleware(path.to_str().unwrap())?; - if middleware.is_none() { + if !matches!(middleware, (Some(_), _)) { middleware = if let Ok(name) = path.file_name_trim_end(".lua") { match name { - "init" | "init.client" | "init.server" => None, - _ => Some(SnapshotMiddleware::Lua), + "init" | "init.client" | "init.server" => (None, None), + _ => { + if let Ok(name) = path.file_name_trim_end(".server.lua") { + (Some(SnapshotMiddleware::LuaServer), Some(name.to_owned())) + } else if let Ok(name) = path.file_name_trim_end(".client.lua") { + (Some(SnapshotMiddleware::LuaClient), Some(name.to_owned())) + } else { + (Some(SnapshotMiddleware::LuaModule), Some(name.to_owned())) + } + } } } else if path.file_name_ends_with(".project.json") { - Some(SnapshotMiddleware::Project) + ( + Some(SnapshotMiddleware::Project), + match path.file_name_trim_end(".project.json") { + Ok(v) => Some(v.to_owned()), + Err(_) => None, + }, + ) } else if path.file_name_ends_with(".model.json") { - Some(SnapshotMiddleware::JsonModel) + ( + Some(SnapshotMiddleware::JsonModel), + match path.file_name_trim_end(".model.json") { + Ok(v) => Some(v.to_owned()), + Err(_) => None, + }, + ) } else if path.file_name_ends_with(".meta.json") { // .meta.json files do not turn into their own instances. - None + (None, None) } else if path.file_name_ends_with(".json") { - Some(SnapshotMiddleware::Json) + ( + Some(SnapshotMiddleware::Json), + match path.file_name_trim_end(".json") { + Ok(v) => Some(v.to_owned()), + Err(_) => None, + }, + ) } else if path.file_name_ends_with(".csv") { - Some(SnapshotMiddleware::Csv) + ( + Some(SnapshotMiddleware::Csv), + match path.file_name_trim_end(".csv") { + Ok(v) => Some(v.to_owned()), + Err(_) => None, + }, + ) } else if path.file_name_ends_with(".txt") { - Some(SnapshotMiddleware::Txt) + ( + Some(SnapshotMiddleware::Txt), + match path.file_name_trim_end(".txt") { + Ok(v) => Some(v.to_owned()), + Err(_) => None, + }, + ) } else if path.file_name_ends_with(".rbxmx") { - Some(SnapshotMiddleware::Rbxmx) + ( + Some(SnapshotMiddleware::Rbxmx), + match path.file_name_trim_end(".rbxmx") { + Ok(v) => Some(v.to_owned()), + Err(_) => None, + }, + ) } else if path.file_name_ends_with(".rbxm") { - Some(SnapshotMiddleware::Rbxm) + ( + Some(SnapshotMiddleware::Rbxm), + match path.file_name_trim_end(".rbxm") { + Ok(v) => Some(v.to_owned()), + Err(_) => None, + }, + ) } else { - None + (None, None) }; } + middleware = match middleware { + // Pick a default name (name without extension) + (Some(x), None) => ( + Some(x), + match path.file_name_no_extension() { + Ok(v) => Some(v.to_owned()), + Err(_) => None, + }, + ), + x => x, + }; + return match middleware { - Some(x) => match x { - SnapshotMiddleware::Lua => snapshot_lua(context, vfs, path), + (Some(x), Some(name)) => match x { + SnapshotMiddleware::LuaModule => { + snapshot_lua(context, vfs, &plugin_env, path, &name, "ModuleScript") + } + SnapshotMiddleware::LuaServer => { + snapshot_lua(context, vfs, &plugin_env, path, &name, "Script") + } + SnapshotMiddleware::LuaClient => { + snapshot_lua(context, vfs, &plugin_env, path, &name, "LocalScript") + } SnapshotMiddleware::Project => snapshot_project(context, vfs, plugin_env, path), SnapshotMiddleware::JsonModel => { - snapshot_json_model(context, vfs, plugin_env, path) + snapshot_json_model(context, vfs, plugin_env, path, &name) } - SnapshotMiddleware::Json => snapshot_json(context, vfs, path), - SnapshotMiddleware::Csv => snapshot_csv(context, vfs, path), - SnapshotMiddleware::Txt => snapshot_txt(context, vfs, path), - SnapshotMiddleware::Rbxmx => snapshot_rbxmx(context, vfs, path), - SnapshotMiddleware::Rbxm => snapshot_rbxm(context, vfs, path), + SnapshotMiddleware::Json => snapshot_json(context, vfs, plugin_env, path, &name), + SnapshotMiddleware::Csv => snapshot_csv(context, vfs, plugin_env, path, &name), + SnapshotMiddleware::Txt => snapshot_txt(context, vfs, plugin_env, path, &name), + SnapshotMiddleware::Rbxmx => snapshot_rbxmx(context, vfs, plugin_env, path, &name), + SnapshotMiddleware::Rbxm => snapshot_rbxm(context, vfs, plugin_env, path, &name), _ => Ok(None), }, - None => Ok(None), + _ => Ok(None), }; } } diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index 3b1b98042..fd6f6905f 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -5,6 +5,7 @@ use memofs::Vfs; use rbx_reflection::ClassTag; use crate::{ + load_file::load_file, plugin_env::PluginEnv, project::{Project, ProjectNode}, snapshot::{ @@ -20,7 +21,8 @@ pub fn snapshot_project( plugin_env: &PluginEnv, path: &Path, ) -> anyhow::Result> { - let project = Project::load_from_slice(&vfs.read(path)?, path) + let contents = load_file(vfs, plugin_env, path)?; + let project = Project::load_from_slice(&contents, path) .with_context(|| format!("File was not a valid Rojo project: {}", path.display()))?; let mut context = context.clone(); @@ -288,6 +290,8 @@ fn infer_class_name(name: &str, parent_class: Option<&str>) -> Option anyhow::Result> { - let name = path.file_name_trim_end(".rbxm")?; - - let temp_tree = rbx_binary::from_reader(vfs.read(path)?.as_slice()) + let contents = load_file(vfs, plugin_env, path)?; + let temp_tree = rbx_binary::from_reader(contents.as_slice()) .with_context(|| format!("Malformed rbxm file: {}", path.display()))?; let root_instance = temp_tree.root(); @@ -42,6 +45,8 @@ pub fn snapshot_rbxm( #[cfg(test)] mod test { + use std::sync::Arc; + use super::*; use memofs::{InMemoryFs, VfsSnapshot}; @@ -55,12 +60,17 @@ mod test { ) .unwrap(); - let mut vfs = Vfs::new(imfs); + let mut vfs = Arc::new(Vfs::new(imfs)); + + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); + plugin_env.init().unwrap(); let instance_snapshot = snapshot_rbxm( &InstanceContext::default(), &mut vfs, + &plugin_env, Path::new("/foo.rbxm"), + "foo", ) .unwrap() .unwrap(); diff --git a/src/snapshot_middleware/rbxmx.rs b/src/snapshot_middleware/rbxmx.rs index 3cdd52e45..efe9b4835 100644 --- a/src/snapshot_middleware/rbxmx.rs +++ b/src/snapshot_middleware/rbxmx.rs @@ -3,21 +3,24 @@ use std::path::Path; use anyhow::Context; use memofs::Vfs; -use crate::snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}; - -use super::util::PathExt; +use crate::{ + load_file::load_file, + plugin_env::PluginEnv, + snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, +}; pub fn snapshot_rbxmx( context: &InstanceContext, vfs: &Vfs, + plugin_env: &PluginEnv, path: &Path, + name: &str, ) -> anyhow::Result> { - let name = path.file_name_trim_end(".rbxmx")?; - let options = rbx_xml::DecodeOptions::new() .property_behavior(rbx_xml::DecodePropertyBehavior::ReadUnknown); - let temp_tree = rbx_xml::from_reader(vfs.read(path)?.as_slice(), options) + let contents = load_file(vfs, plugin_env, path)?; + let temp_tree = rbx_xml::from_reader(contents.as_slice(), options) .with_context(|| format!("Malformed rbxm file: {}", path.display()))?; let root_instance = temp_tree.root(); @@ -45,6 +48,8 @@ pub fn snapshot_rbxmx( #[cfg(test)] mod test { + use std::sync::Arc; + use super::*; use memofs::{InMemoryFs, VfsSnapshot}; @@ -68,12 +73,17 @@ mod test { ) .unwrap(); - let mut vfs = Vfs::new(imfs); + let mut vfs = Arc::new(Vfs::new(imfs)); + + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); + plugin_env.init().unwrap(); let instance_snapshot = snapshot_rbxmx( &InstanceContext::default(), &mut vfs, + &plugin_env, Path::new("/foo.rbxmx"), + "foo", ) .unwrap() .unwrap(); diff --git a/src/snapshot_middleware/txt.rs b/src/snapshot_middleware/txt.rs index 13d5b9907..5cac2cb65 100644 --- a/src/snapshot_middleware/txt.rs +++ b/src/snapshot_middleware/txt.rs @@ -4,18 +4,22 @@ use anyhow::Context; use maplit::hashmap; use memofs::{IoResultExt, Vfs}; -use crate::snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}; +use crate::{ + load_file::load_file, + plugin_env::PluginEnv, + snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, +}; -use super::{meta_file::AdjacentMetadata, util::PathExt}; +use super::meta_file::AdjacentMetadata; pub fn snapshot_txt( context: &InstanceContext, vfs: &Vfs, + plugin_env: &PluginEnv, path: &Path, + name: &str, ) -> anyhow::Result> { - let name = path.file_name_trim_end(".txt")?; - - let contents = vfs.read(path)?; + let contents = load_file(vfs, plugin_env, path)?; let contents_str = str::from_utf8(&contents) .with_context(|| format!("File was not valid UTF-8: {}", path.display()))? .to_owned(); @@ -47,6 +51,8 @@ pub fn snapshot_txt( #[cfg(test)] mod test { + use std::sync::Arc; + use super::*; use memofs::{InMemoryFs, VfsSnapshot}; @@ -57,12 +63,20 @@ mod test { imfs.load_snapshot("/foo.txt", VfsSnapshot::file("Hello there!")) .unwrap(); - let mut vfs = Vfs::new(imfs.clone()); + let mut vfs = Arc::new(Vfs::new(imfs)); + + let plugin_env = PluginEnv::new(Arc::clone(&vfs)); + plugin_env.init().unwrap(); - let instance_snapshot = - snapshot_txt(&InstanceContext::default(), &mut vfs, Path::new("/foo.txt")) - .unwrap() - .unwrap(); + let instance_snapshot = snapshot_txt( + &InstanceContext::default(), + &mut vfs, + &plugin_env, + Path::new("/foo.txt"), + "foo", + ) + .unwrap() + .unwrap(); insta::assert_yaml_snapshot!(instance_snapshot); } diff --git a/src/snapshot_middleware/util.rs b/src/snapshot_middleware/util.rs index 517cd6066..6cf439226 100644 --- a/src/snapshot_middleware/util.rs +++ b/src/snapshot_middleware/util.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::{ops::Index, path::Path}; use anyhow::Context; @@ -16,6 +16,7 @@ pub fn match_trailing<'a>(input: &'a str, suffix: &str) -> Option<&'a str> { pub trait PathExt { fn file_name_ends_with(&self, suffix: &str) -> bool; fn file_name_trim_end<'a>(&'a self, suffix: &str) -> anyhow::Result<&'a str>; + fn file_name_no_extension<'a>(&'a self) -> anyhow::Result<&'a str>; } impl

PathExt for P @@ -40,4 +41,15 @@ where match_trailing(&file_name, suffix) .with_context(|| format!("Path did not end in {}: {}", suffix, path.display())) } + + fn file_name_no_extension<'a>(&'a self) -> anyhow::Result<&'a str> { + let path = self.as_ref(); + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .with_context(|| format!("Path did not have a file name: {}", path.display()))?; + + let index = file_name.chars().position(|c| c == '.').unwrap_or(0); + Ok(&file_name[0..index]) + } } diff --git a/test-projects/plugins/default.project.json b/test-projects/plugins/default.project.json index d80bdd8ae..61bdc59f3 100644 --- a/test-projects/plugins/default.project.json +++ b/test-projects/plugins/default.project.json @@ -1,6 +1,7 @@ { "name": "plugins", "plugins": [ + "fake-moonscript.lua", { "source": "load-as-stringvalue.lua", "options": { "extensions": ["md"] } diff --git a/test-projects/plugins/fake-moonscript.lua b/test-projects/plugins/fake-moonscript.lua new file mode 100644 index 000000000..5910875ed --- /dev/null +++ b/test-projects/plugins/fake-moonscript.lua @@ -0,0 +1,39 @@ +print('[plugin(fake-moonscript)] loading') + +-- This does not actually compile moonscript, it is just to test the hooks that would be used for a +-- real one. + +local function compile(moonscript) + return moonscript +end + +return function(options) + print('[plugin(fake-moonscript)] create') + + return { + name = 'fake-moonscript', + middleware = function(id) + print(('[plugin(fake-moonscript)] middleware: %s'):format(id)) + if id:match('%.moon$') then + print('[plugin(fake-moonscript)] matched') + if id:match('%.server%.moon$') then + return 'lua_server' + elseif id:match('%.client%.moon$') then + return 'lua_client' + else + return 'lua_module' + end + end + print('[plugin(fake-moonscript)] skipping') + end, + load = function(id) + print(('[plugin(fake-moonscript)] load: %s'):format(id)) + if id:match('%.moon$') then + print('[plugin(fake-moonscript)] matched') + local contents = rojo.readFileAsUtf8(id) + return compile(contents) + end + print('[plugin(fake-moonscript)] skipping') + end + } +end diff --git a/test-projects/plugins/load-as-stringvalue.lua b/test-projects/plugins/load-as-stringvalue.lua index 71d4ab16f..668429d26 100644 --- a/test-projects/plugins/load-as-stringvalue.lua +++ b/test-projects/plugins/load-as-stringvalue.lua @@ -1,4 +1,4 @@ -print('[plugin] loading: load-as-stringvalue.lua') +print('[plugin(load-as-stringvalue)] loading') local function tableToString(t) local s = '' @@ -21,34 +21,34 @@ local function tableToString(t) end return function(options) - print(('[plugin] create with: %s'):format(tableToString(options))) + print(('[plugin(load-as-stringvalue)] create with: %s'):format(tableToString(options))) options.extensions = options.extensions or {} return { name = 'load-as-stringvalue', middleware = function(id) - print(('[plugin] middleware: %s'):format(id)) + print(('[plugin(load-as-stringvalue)] middleware: %s'):format(id)) local idExt = id:match('%.(%w+)$') for _, ext in next, options.extensions do if ext == idExt then - print(('[plugin] matched: %s'):format(ext)) + print(('[plugin(load-as-stringvalue)] matched: %s'):format(ext)) return 'json_model' end end - print('[plugin] skipping') + print('[plugin(load-as-stringvalue)] skipping') end, load = function(id) - print(('[plugin] load: %s'):format(id)) + print(('[plugin(load-as-stringvalue)] load: %s'):format(id)) local idExt = id:match('%.(%w+)$') for _, ext in next, options.extensions do if ext == idExt then local contents = rojo.readFileAsUtf8(id) - print(('[plugin] matched: %s'):format(ext)) + print(('[plugin(load-as-stringvalue)] matched: %s'):format(ext)) local encoded = contents:gsub('\n', '\\n') return ('{"ClassName": "StringValue", "Properties": { "Value": "%s" }}'):format(encoded) end end - print('[plugin] skipping') + print('[plugin(load-as-stringvalue)] skipping') end } end diff --git a/test-projects/plugins/src/Another.md b/test-projects/plugins/src/Another.md deleted file mode 100644 index 56323ef55..000000000 --- a/test-projects/plugins/src/Another.md +++ /dev/null @@ -1 +0,0 @@ -Some **bold** text. diff --git a/test-projects/plugins/src/Document.md b/test-projects/plugins/src/Document.md new file mode 100644 index 000000000..6e2652bd1 --- /dev/null +++ b/test-projects/plugins/src/Document.md @@ -0,0 +1,3 @@ +# Document + +A **bold** statement made in Markdown. diff --git a/test-projects/plugins/src/Hello.client.moon b/test-projects/plugins/src/Hello.client.moon new file mode 100644 index 000000000..f8fce2672 --- /dev/null +++ b/test-projects/plugins/src/Hello.client.moon @@ -0,0 +1 @@ +print "Hello from the client!" \ No newline at end of file diff --git a/test-projects/plugins/src/Hello.md b/test-projects/plugins/src/Hello.md deleted file mode 100644 index f01722bb1..000000000 --- a/test-projects/plugins/src/Hello.md +++ /dev/null @@ -1,3 +0,0 @@ -# Markdown - -WooHoo! diff --git a/test-projects/plugins/src/Hello.moon b/test-projects/plugins/src/Hello.moon new file mode 100644 index 000000000..057da09bb --- /dev/null +++ b/test-projects/plugins/src/Hello.moon @@ -0,0 +1 @@ +return "Hello from a module!" diff --git a/test-projects/plugins/src/Hello.server.moon b/test-projects/plugins/src/Hello.server.moon new file mode 100644 index 000000000..7526d7267 --- /dev/null +++ b/test-projects/plugins/src/Hello.server.moon @@ -0,0 +1 @@ +print "Hello from the server!" From fa31815aad86a3fc4c2f7b1323cbd25b71f402bc Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 17:08:39 -0500 Subject: [PATCH 18/22] added todo --- src/snapshot_middleware/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index c902d4332..afff70750 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -90,6 +90,7 @@ pub fn snapshot_from_vfs( None => return Ok(None), }; + // TODO: Think about how to handle this stuff for plugins. if meta.is_dir() { let project_path = path.join("default.project.json"); if vfs.metadata(&project_path).with_not_found()?.is_some() { From 07cd4413d992db8aa5a9115ce6d7b1e7f0bcd24a Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 17:21:57 -0500 Subject: [PATCH 19/22] add description of middleware usage --- PLUGIN-DESIGN.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/PLUGIN-DESIGN.md b/PLUGIN-DESIGN.md index ffcc6189d..d20a8f024 100644 --- a/PLUGIN-DESIGN.md +++ b/PLUGIN-DESIGN.md @@ -16,6 +16,12 @@ primary Rollup config file. Rollup then calls in to plugin "hooks" at special ti bundling process. I have found Rollup plugins to be an excellent interface for both plugin developers and end-users and therefore I have based this proposal on their API. +This proposal attempts to take advantage of the existing "middleware" system in Rojo, and give +plugins the opportunity to: + +- Override the default choice for which internal middleware should be used for a given file path +- Transform the contents of a file before consuming it + ## Project file changes Add a new top-level field to the project file format: From 9e28502324979292547bf49edd5b81355549292e Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 17:31:06 -0500 Subject: [PATCH 20/22] import plugins relative to project file --- src/serve_session.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/serve_session.rs b/src/serve_session.rs index a5a4d6e34..35b442d14 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -159,7 +159,11 @@ impl ServeSession { } }; - match plugin_env.load_plugin(&plugin_source, plugin_options) { + let temp = project_path.with_file_name(plugin_source); + let plugin_source_path = temp.to_str().unwrap(); + println!("{}", plugin_source_path); + + match plugin_env.load_plugin(plugin_source_path, plugin_options) { Ok(_) => (), Err(e) => return Err(ServeSessionError::Plugin { source: e }), }; From c324500b90ae64fc2a6af11476e3e5d8fc0486c3 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 17:37:30 -0500 Subject: [PATCH 21/22] fixed middleware returns for example moonscript --- PLUGIN-DESIGN.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/PLUGIN-DESIGN.md b/PLUGIN-DESIGN.md index d20a8f024..1cf152c06 100644 --- a/PLUGIN-DESIGN.md +++ b/PLUGIN-DESIGN.md @@ -164,7 +164,13 @@ return function(options) name = "moonscript", middleware = function(id) if rojo.hasExtension(id, 'moon') then - return 'lua' + if rojo.hasExtension(id, 'server.moon') then + return 'lua_server' + elseif rojo.hasExtension(id, 'client.moon') then + return 'lua_client' + else + return 'lua_module' + end end end, load = function(id) From 3cb03d39712efc38955d72864c7aef2656e6c5d3 Mon Sep 17 00:00:00 2001 From: Blake Mealey Date: Thu, 28 Oct 2021 17:37:38 -0500 Subject: [PATCH 22/22] removed debug print --- src/serve_session.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/serve_session.rs b/src/serve_session.rs index 35b442d14..ff5239d5d 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -161,7 +161,6 @@ impl ServeSession { let temp = project_path.with_file_name(plugin_source); let plugin_source_path = temp.to_str().unwrap(); - println!("{}", plugin_source_path); match plugin_env.load_plugin(plugin_source_path, plugin_options) { Ok(_) => (),