diff --git a/Pipfile b/Pipfile
index 318e601fb..9543c1123 100644
--- a/Pipfile
+++ b/Pipfile
@@ -50,6 +50,7 @@ tiktoken = "==0.5.1"
 hf-transfer = "==0.1.3"
 peft = "==0.5.0"
 azure-storage-file-datalake = ">=12.12.0"
+deepspeed = "==0.11.1"
 keyring = "==24.2.0"
 
 [dev-packages]
diff --git a/Pipfile.lock b/Pipfile.lock
index 33c5f14fd..c52b47b92 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "4dc5edfa5dbc0ed27db4dca77b3109318f89ffd8d06c1b151ebcb0e5dec77034"
+            "sha256": "d6da98ab1ca6c30933bd9ed0d472f64449b554aa87ed524c61c32974a497c71a"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -31,96 +31,96 @@
         },
         "aiohttp": {
             "hashes": [
-                "sha256:00ad4b6f185ec67f3e6562e8a1d2b69660be43070bd0ef6fcec5211154c7df67",
-                "sha256:0175d745d9e85c40dcc51c8f88c74bfbaef9e7afeeeb9d03c37977270303064c",
-                "sha256:01d4c0c874aa4ddfb8098e85d10b5e875a70adc63db91f1ae65a4b04d3344cda",
-                "sha256:043d2299f6dfdc92f0ac5e995dfc56668e1587cea7f9aa9d8a78a1b6554e5755",
-                "sha256:0c413c633d0512df4dc7fd2373ec06cc6a815b7b6d6c2f208ada7e9e93a5061d",
-                "sha256:0d21c684808288a98914e5aaf2a7c6a3179d4df11d249799c32d1808e79503b5",
-                "sha256:0e584a10f204a617d71d359fe383406305a4b595b333721fa50b867b4a0a1548",
-                "sha256:1274477e4c71ce8cfe6c1ec2f806d57c015ebf84d83373676036e256bc55d690",
-                "sha256:13bf85afc99ce6f9ee3567b04501f18f9f8dbbb2ea11ed1a2e079670403a7c84",
-                "sha256:153c2549f6c004d2754cc60603d4668899c9895b8a89397444a9c4efa282aaf4",
-                "sha256:1f7372f7341fcc16f57b2caded43e81ddd18df53320b6f9f042acad41f8e049a",
-                "sha256:23fb25a9f0a1ca1f24c0a371523546366bb642397c94ab45ad3aedf2941cec6a",
-                "sha256:28c543e54710d6158fc6f439296c7865b29e0b616629767e685a7185fab4a6b9",
-                "sha256:2a482e6da906d5e6e653be079b29bc173a48e381600161c9932d89dfae5942ef",
-                "sha256:2ad5c3c4590bb3cc28b4382f031f3783f25ec223557124c68754a2231d989e2b",
-                "sha256:2ce2ac5708501afc4847221a521f7e4b245abf5178cf5ddae9d5b3856ddb2f3a",
-                "sha256:2cf57fb50be5f52bda004b8893e63b48530ed9f0d6c96c84620dc92fe3cd9b9d",
-                "sha256:2e1b1e51b0774408f091d268648e3d57f7260c1682e7d3a63cb00d22d71bb945",
-                "sha256:2e2e9839e14dd5308ee773c97115f1e0a1cb1d75cbeeee9f33824fa5144c7634",
-                "sha256:2e460be6978fc24e3df83193dc0cc4de46c9909ed92dd47d349a452ef49325b7",
-                "sha256:312fcfbacc7880a8da0ae8b6abc6cc7d752e9caa0051a53d217a650b25e9a691",
-                "sha256:33279701c04351a2914e1100b62b2a7fdb9a25995c4a104259f9a5ead7ed4802",
-                "sha256:33776e945d89b29251b33a7e7d006ce86447b2cfd66db5e5ded4e5cd0340585c",
-                "sha256:34dd0c107799dcbbf7d48b53be761a013c0adf5571bf50c4ecad5643fe9cfcd0",
-                "sha256:3562b06567c06439d8b447037bb655ef69786c590b1de86c7ab81efe1c9c15d8",
-                "sha256:368a42363c4d70ab52c2c6420a57f190ed3dfaca6a1b19afda8165ee16416a82",
-                "sha256:4149d34c32f9638f38f544b3977a4c24052042affa895352d3636fa8bffd030a",
-                "sha256:461908b2578955045efde733719d62f2b649c404189a09a632d245b445c9c975",
-                "sha256:4a01951fabc4ce26ab791da5f3f24dca6d9a6f24121746eb19756416ff2d881b",
-                "sha256:4e874cbf8caf8959d2adf572a78bba17cb0e9d7e51bb83d86a3697b686a0ab4d",
-                "sha256:4f21e83f355643c345177a5d1d8079f9f28b5133bcd154193b799d380331d5d3",
-                "sha256:5443910d662db951b2e58eb70b0fbe6b6e2ae613477129a5805d0b66c54b6cb7",
-                "sha256:5798a9aad1879f626589f3df0f8b79b3608a92e9beab10e5fda02c8a2c60db2e",
-                "sha256:5d20003b635fc6ae3f96d7260281dfaf1894fc3aa24d1888a9b2628e97c241e5",
-                "sha256:5db3a5b833764280ed7618393832e0853e40f3d3e9aa128ac0ba0f8278d08649",
-                "sha256:5ed1c46fb119f1b59304b5ec89f834f07124cd23ae5b74288e364477641060ff",
-                "sha256:62360cb771707cb70a6fd114b9871d20d7dd2163a0feafe43fd115cfe4fe845e",
-                "sha256:6809a00deaf3810e38c628e9a33271892f815b853605a936e2e9e5129762356c",
-                "sha256:68c5a82c8779bdfc6367c967a4a1b2aa52cd3595388bf5961a62158ee8a59e22",
-                "sha256:6e4a280e4b975a2e7745573e3fc9c9ba0d1194a3738ce1cbaa80626cc9b4f4df",
-                "sha256:6e6783bcc45f397fdebc118d772103d751b54cddf5b60fbcc958382d7dd64f3e",
-                "sha256:72a860c215e26192379f57cae5ab12b168b75db8271f111019509a1196dfc780",
-                "sha256:7607ec3ce4993464368505888af5beb446845a014bc676d349efec0e05085905",
-                "sha256:773dd01706d4db536335fcfae6ea2440a70ceb03dd3e7378f3e815b03c97ab51",
-                "sha256:78d847e4cde6ecc19125ccbc9bfac4a7ab37c234dd88fbb3c5c524e8e14da543",
-                "sha256:7dde0009408969a43b04c16cbbe252c4f5ef4574ac226bc8815cd7342d2028b6",
-                "sha256:80bd372b8d0715c66c974cf57fe363621a02f359f1ec81cba97366948c7fc873",
-                "sha256:841cd8233cbd2111a0ef0a522ce016357c5e3aff8a8ce92bcfa14cef890d698f",
-                "sha256:84de26ddf621d7ac4c975dbea4c945860e08cccde492269db4e1538a6a6f3c35",
-                "sha256:84f8ae3e09a34f35c18fa57f015cc394bd1389bce02503fb30c394d04ee6b938",
-                "sha256:8af740fc2711ad85f1a5c034a435782fbd5b5f8314c9a3ef071424a8158d7f6b",
-                "sha256:8b929b9bd7cd7c3939f8bcfffa92fae7480bd1aa425279d51a89327d600c704d",
-                "sha256:910bec0c49637d213f5d9877105d26e0c4a4de2f8b1b29405ff37e9fc0ad52b8",
-                "sha256:96943e5dcc37a6529d18766597c491798b7eb7a61d48878611298afc1fca946c",
-                "sha256:a0215ce6041d501f3155dc219712bc41252d0ab76474615b9700d63d4d9292af",
-                "sha256:a3cf433f127efa43fee6b90ea4c6edf6c4a17109d1d037d1a52abec84d8f2e42",
-                "sha256:a6ce61195c6a19c785df04e71a4537e29eaa2c50fe745b732aa937c0c77169f3",
-                "sha256:a7a75ef35f2df54ad55dbf4b73fe1da96f370e51b10c91f08b19603c64004acc",
-                "sha256:a94159871304770da4dd371f4291b20cac04e8c94f11bdea1c3478e557fbe0d8",
-                "sha256:aa1990247f02a54185dc0dff92a6904521172a22664c863a03ff64c42f9b5410",
-                "sha256:ab88bafedc57dd0aab55fa728ea10c1911f7e4d8b43e1d838a1739f33712921c",
-                "sha256:ad093e823df03bb3fd37e7dec9d4670c34f9e24aeace76808fc20a507cace825",
-                "sha256:ae871a964e1987a943d83d6709d20ec6103ca1eaf52f7e0d36ee1b5bebb8b9b9",
-                "sha256:b0ba0d15164eae3d878260d4c4df859bbdc6466e9e6689c344a13334f988bb53",
-                "sha256:b5411d82cddd212644cf9360879eb5080f0d5f7d809d03262c50dad02f01421a",
-                "sha256:b9552ec52cc147dbf1944ac7ac98af7602e51ea2dcd076ed194ca3c0d1c7d0bc",
-                "sha256:bfb9162dcf01f615462b995a516ba03e769de0789de1cadc0f916265c257e5d8",
-                "sha256:c0a9034379a37ae42dea7ac1e048352d96286626251862e448933c0f59cbd79c",
-                "sha256:c1161b345c0a444ebcf46bf0a740ba5dcf50612fd3d0528883fdc0eff578006a",
-                "sha256:c11f5b099adafb18e65c2c997d57108b5bbeaa9eeee64a84302c0978b1ec948b",
-                "sha256:c44e65da1de4403d0576473e2344828ef9c4c6244d65cf4b75549bb46d40b8dd",
-                "sha256:c48c5c0271149cfe467c0ff8eb941279fd6e3f65c9a388c984e0e6cf57538e14",
-                "sha256:c7a815258e5895d8900aec4454f38dca9aed71085f227537208057853f9d13f2",
-                "sha256:cae533195e8122584ec87531d6df000ad07737eaa3c81209e85c928854d2195c",
-                "sha256:cc14be025665dba6202b6a71cfcdb53210cc498e50068bc088076624471f8bb9",
-                "sha256:cd56db019015b6acfaaf92e1ac40eb8434847d9bf88b4be4efe5bfd260aee692",
-                "sha256:d827176898a2b0b09694fbd1088c7a31836d1a505c243811c87ae53a3f6273c1",
-                "sha256:df72ac063b97837a80d80dec8d54c241af059cc9bb42c4de68bd5b61ceb37caa",
-                "sha256:e5980a746d547a6ba173fd5ee85ce9077e72d118758db05d229044b469d9029a",
-                "sha256:e5d47ae48db0b2dcf70bc8a3bc72b3de86e2a590fc299fdbbb15af320d2659de",
-                "sha256:e91d635961bec2d8f19dfeb41a539eb94bd073f075ca6dae6c8dc0ee89ad6f91",
-                "sha256:ea353162f249c8097ea63c2169dd1aa55de1e8fecbe63412a9bc50816e87b761",
-                "sha256:eaeed7abfb5d64c539e2db173f63631455f1196c37d9d8d873fc316470dfbacd",
-                "sha256:eca4bf3734c541dc4f374ad6010a68ff6c6748f00451707f39857f429ca36ced",
-                "sha256:f83a552443a526ea38d064588613aca983d0ee0038801bc93c0c916428310c28",
-                "sha256:fb1558def481d84f03b45888473fc5a1f35747b5f334ef4e7a571bc0dfcb11f8",
-                "sha256:fd1ed388ea7fbed22c4968dd64bab0198de60750a25fe8c0c9d4bef5abe13824"
+                "sha256:002f23e6ea8d3dd8d149e569fd580c999232b5fbc601c48d55398fbc2e582e8c",
+                "sha256:01770d8c04bd8db568abb636c1fdd4f7140b284b8b3e0b4584f070180c1e5c62",
+                "sha256:0912ed87fee967940aacc5306d3aa8ba3a459fcd12add0b407081fbefc931e53",
+                "sha256:0cccd1de239afa866e4ce5c789b3032442f19c261c7d8a01183fd956b1935349",
+                "sha256:0fa375b3d34e71ccccf172cab401cd94a72de7a8cc01847a7b3386204093bb47",
+                "sha256:13da35c9ceb847732bf5c6c5781dcf4780e14392e5d3b3c689f6d22f8e15ae31",
+                "sha256:14cd52ccf40006c7a6cd34a0f8663734e5363fd981807173faf3a017e202fec9",
+                "sha256:16d330b3b9db87c3883e565340d292638a878236418b23cc8b9b11a054aaa887",
+                "sha256:1bed815f3dc3d915c5c1e556c397c8667826fbc1b935d95b0ad680787896a358",
+                "sha256:1d84166673694841d8953f0a8d0c90e1087739d24632fe86b1a08819168b4566",
+                "sha256:1f13f60d78224f0dace220d8ab4ef1dbc37115eeeab8c06804fec11bec2bbd07",
+                "sha256:229852e147f44da0241954fc6cb910ba074e597f06789c867cb7fb0621e0ba7a",
+                "sha256:253bf92b744b3170eb4c4ca2fa58f9c4b87aeb1df42f71d4e78815e6e8b73c9e",
+                "sha256:255ba9d6d5ff1a382bb9a578cd563605aa69bec845680e21c44afc2670607a95",
+                "sha256:2817b2f66ca82ee699acd90e05c95e79bbf1dc986abb62b61ec8aaf851e81c93",
+                "sha256:2b8d4e166e600dcfbff51919c7a3789ff6ca8b3ecce16e1d9c96d95dd569eb4c",
+                "sha256:2d5b785c792802e7b275c420d84f3397668e9d49ab1cb52bd916b3b3ffcf09ad",
+                "sha256:3161ce82ab85acd267c8f4b14aa226047a6bee1e4e6adb74b798bd42c6ae1f80",
+                "sha256:33164093be11fcef3ce2571a0dccd9041c9a93fa3bde86569d7b03120d276c6f",
+                "sha256:39a312d0e991690ccc1a61f1e9e42daa519dcc34ad03eb6f826d94c1190190dd",
+                "sha256:3b2ab182fc28e7a81f6c70bfbd829045d9480063f5ab06f6e601a3eddbbd49a0",
+                "sha256:3c68330a59506254b556b99a91857428cab98b2f84061260a67865f7f52899f5",
+                "sha256:3f0e27e5b733803333bb2371249f41cf42bae8884863e8e8965ec69bebe53132",
+                "sha256:3f5c7ce535a1d2429a634310e308fb7d718905487257060e5d4598e29dc17f0b",
+                "sha256:3fd194939b1f764d6bb05490987bfe104287bbf51b8d862261ccf66f48fb4096",
+                "sha256:41bdc2ba359032e36c0e9de5a3bd00d6fb7ea558a6ce6b70acedf0da86458321",
+                "sha256:41d55fc043954cddbbd82503d9cc3f4814a40bcef30b3569bc7b5e34130718c1",
+                "sha256:42c89579f82e49db436b69c938ab3e1559e5a4409eb8639eb4143989bc390f2f",
+                "sha256:45ad816b2c8e3b60b510f30dbd37fe74fd4a772248a52bb021f6fd65dff809b6",
+                "sha256:4ac39027011414dbd3d87f7edb31680e1f430834c8cef029f11c66dad0670aa5",
+                "sha256:4d4cbe4ffa9d05f46a28252efc5941e0462792930caa370a6efaf491f412bc66",
+                "sha256:4fcf3eabd3fd1a5e6092d1242295fa37d0354b2eb2077e6eb670accad78e40e1",
+                "sha256:5d791245a894be071d5ab04bbb4850534261a7d4fd363b094a7b9963e8cdbd31",
+                "sha256:6c43ecfef7deaf0617cee936836518e7424ee12cb709883f2c9a1adda63cc460",
+                "sha256:6c5f938d199a6fdbdc10bbb9447496561c3a9a565b43be564648d81e1102ac22",
+                "sha256:6e2f9cc8e5328f829f6e1fb74a0a3a939b14e67e80832975e01929e320386b34",
+                "sha256:713103a8bdde61d13490adf47171a1039fd880113981e55401a0f7b42c37d071",
+                "sha256:71783b0b6455ac8f34b5ec99d83e686892c50498d5d00b8e56d47f41b38fbe04",
+                "sha256:76b36b3124f0223903609944a3c8bf28a599b2cc0ce0be60b45211c8e9be97f8",
+                "sha256:7bc88fc494b1f0311d67f29fee6fd636606f4697e8cc793a2d912ac5b19aa38d",
+                "sha256:7ee912f7e78287516df155f69da575a0ba33b02dd7c1d6614dbc9463f43066e3",
+                "sha256:86f20cee0f0a317c76573b627b954c412ea766d6ada1a9fcf1b805763ae7feeb",
+                "sha256:89341b2c19fb5eac30c341133ae2cc3544d40d9b1892749cdd25892bbc6ac951",
+                "sha256:8a9b5a0606faca4f6cc0d338359d6fa137104c337f489cd135bb7fbdbccb1e39",
+                "sha256:8d399dade330c53b4106160f75f55407e9ae7505263ea86f2ccca6bfcbdb4921",
+                "sha256:8e31e9db1bee8b4f407b77fd2507337a0a80665ad7b6c749d08df595d88f1cf5",
+                "sha256:90c72ebb7cb3a08a7f40061079817133f502a160561d0675b0a6adf231382c92",
+                "sha256:918810ef188f84152af6b938254911055a72e0f935b5fbc4c1a4ed0b0584aed1",
+                "sha256:93c15c8e48e5e7b89d5cb4613479d144fda8344e2d886cf694fd36db4cc86865",
+                "sha256:96603a562b546632441926cd1293cfcb5b69f0b4159e6077f7c7dbdfb686af4d",
+                "sha256:99c5ac4ad492b4a19fc132306cd57075c28446ec2ed970973bbf036bcda1bcc6",
+                "sha256:9c19b26acdd08dd239e0d3669a3dddafd600902e37881f13fbd8a53943079dbc",
+                "sha256:9de50a199b7710fa2904be5a4a9b51af587ab24c8e540a7243ab737b45844543",
+                "sha256:9e2ee0ac5a1f5c7dd3197de309adfb99ac4617ff02b0603fd1e65b07dc772e4b",
+                "sha256:a2ece4af1f3c967a4390c284797ab595a9f1bc1130ef8b01828915a05a6ae684",
+                "sha256:a3628b6c7b880b181a3ae0a0683698513874df63783fd89de99b7b7539e3e8a8",
+                "sha256:ad1407db8f2f49329729564f71685557157bfa42b48f4b93e53721a16eb813ed",
+                "sha256:b04691bc6601ef47c88f0255043df6f570ada1a9ebef99c34bd0b72866c217ae",
+                "sha256:b0cf2a4501bff9330a8a5248b4ce951851e415bdcce9dc158e76cfd55e15085c",
+                "sha256:b2fe42e523be344124c6c8ef32a011444e869dc5f883c591ed87f84339de5976",
+                "sha256:b30e963f9e0d52c28f284d554a9469af073030030cef8693106d918b2ca92f54",
+                "sha256:bb54c54510e47a8c7c8e63454a6acc817519337b2b78606c4e840871a3e15349",
+                "sha256:bd111d7fc5591ddf377a408ed9067045259ff2770f37e2d94e6478d0f3fc0c17",
+                "sha256:bdf70bfe5a1414ba9afb9d49f0c912dc524cf60141102f3a11143ba3d291870f",
+                "sha256:ca80e1b90a05a4f476547f904992ae81eda5c2c85c66ee4195bb8f9c5fb47f28",
+                "sha256:caf486ac1e689dda3502567eb89ffe02876546599bbf915ec94b1fa424eeffd4",
+                "sha256:ccc360e87341ad47c777f5723f68adbb52b37ab450c8bc3ca9ca1f3e849e5fe2",
+                "sha256:d25036d161c4fe2225d1abff2bd52c34ed0b1099f02c208cd34d8c05729882f0",
+                "sha256:d52d5dc7c6682b720280f9d9db41d36ebe4791622c842e258c9206232251ab2b",
+                "sha256:d67f8baed00870aa390ea2590798766256f31dc5ed3ecc737debb6e97e2ede78",
+                "sha256:d76e8b13161a202d14c9584590c4df4d068c9567c99506497bdd67eaedf36403",
+                "sha256:d95fc1bf33a9a81469aa760617b5971331cdd74370d1214f0b3109272c0e1e3c",
+                "sha256:de6a1c9f6803b90e20869e6b99c2c18cef5cc691363954c93cb9adeb26d9f3ae",
+                "sha256:e1d8cb0b56b3587c5c01de3bf2f600f186da7e7b5f7353d1bf26a8ddca57f965",
+                "sha256:e2a988a0c673c2e12084f5e6ba3392d76c75ddb8ebc6c7e9ead68248101cd446",
+                "sha256:e3f1e3f1a1751bb62b4a1b7f4e435afcdade6c17a4fd9b9d43607cebd242924a",
+                "sha256:e6a00ffcc173e765e200ceefb06399ba09c06db97f401f920513a10c803604ca",
+                "sha256:e827d48cf802de06d9c935088c2924e3c7e7533377d66b6f31ed175c1620e05e",
+                "sha256:ebf3fd9f141700b510d4b190094db0ce37ac6361a6806c153c161dc6c041ccda",
+                "sha256:ec00c3305788e04bf6d29d42e504560e159ccaf0be30c09203b468a6c1ccd3b2",
+                "sha256:ec4fd86658c6a8964d75426517dc01cbf840bbf32d055ce64a9e63a40fd7b771",
+                "sha256:efd2fcf7e7b9d7ab16e6b7d54205beded0a9c8566cb30f09c1abe42b4e22bdcb",
+                "sha256:f0f03211fd14a6a0aed2997d4b1c013d49fb7b50eeb9ffdf5e51f23cfe2c77fa",
+                "sha256:f628dbf3c91e12f4d6c8b3f092069567d8eb17814aebba3d7d60c149391aee3a",
+                "sha256:f8ef51e459eb2ad8e7a66c1d6440c808485840ad55ecc3cafefadea47d1b1ba2",
+                "sha256:fc37e9aef10a696a5a4474802930079ccfc14d9f9c10b4662169671ff034b7df",
+                "sha256:fdee8405931b0615220e5ddf8cd7edd8592c606a8e4ca2a00704883c396e4479"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==3.8.5"
+            "version": "==3.8.6"
         },
         "aiosignal": {
             "hashes": [
@@ -172,19 +172,19 @@
         },
         "azure-storage-blob": {
             "hashes": [
-                "sha256:e11935348981ffc005b848b55db25c04f2d1f90e1ee33000659906b763cf14c8",
-                "sha256:ffd864bf9abf33dfc72c6ef37899a19bd9d585a946a2c61e288b4420c035df3a"
+                "sha256:c278dde2ac41857a68d615c9f2b36d894ba877a7e84d62795603c7e79d0bb5e9",
+                "sha256:d8ced0deee3367fa3d4f3d1a03cd9edadf4440c0a371f503d623fa6c807554ee"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==12.18.2"
+            "version": "==12.18.3"
         },
         "azure-storage-file-datalake": {
             "hashes": [
-                "sha256:54e8b19e8e3d83bb5fbc51c3a17ed78d6be2811cc78af12beadc5b547bba574e",
-                "sha256:e53dbbfef817fac63de0df92787d8b84798d7f1e12d9d216f30b722b6b8ccf31"
+                "sha256:04c043bac0a996184eac043c42aecde880f9cebf2811af158f33e384458e8037",
+                "sha256:7b199828b1abe170d64d68d67e0b74200a9a445a2261c1a96fdd06f584d34f36"
             ],
             "index": "pypi",
-            "version": "==12.13.1"
+            "version": "==12.13.2"
         },
         "beautifulsoup4": {
             "hashes": [
@@ -204,11 +204,11 @@
         },
         "bleach": {
             "hashes": [
-                "sha256:1a1a85c1595e07d8db14c5f09f09e6433502c51c595970edc090551f0db99414",
-                "sha256:33c16e3353dbd13028ab4799a0f89a83f113405c766e9c122df8a06f5b85b3f4"
+                "sha256:0a31f1837963c41d46bbf1331b8778e1308ea0791db03cc4e7357b97cf42a8fe",
+                "sha256:3225f354cfc436b9789c66c4ee030194bee0568fbf9cbdad3bc8b5c26c5f12b6"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==6.0.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==6.1.0"
         },
         "blessed": {
             "hashes": [
@@ -220,27 +220,27 @@
         },
         "bokeh": {
             "hashes": [
-                "sha256:b2959b8524d69ec4e7886bc36407445f0a92e1f19530d3bfc4045236a1b7a6ff",
-                "sha256:e31670a013e1ff15c3d4d04f587c8162c4cc9fe1af507fd741d295e6c4e1225b"
+                "sha256:65e36824c99fd46530c559263c6d14eabed6945370cd3beffc2eeedb62d6db6d",
+                "sha256:cdbe268f842c139ba8fa0fb43c0c55c172c8215ec5a69a2629482c63c9d4039c"
             ],
             "index": "pypi",
-            "version": "==3.2.2"
+            "version": "==3.3.0"
         },
         "boto3": {
             "hashes": [
-                "sha256:7a539aaf00eb45aea1ae857ef5d05e67def24fc07af4cb36c202fa45f8f30590",
-                "sha256:ec49986e6c9549177e351494de64886c3f9daffd1a7af9e40302208aa1ffff1c"
+                "sha256:9d52a1605657aeb5b19b09cfc01d9a92f88a616a5daf5479a59656d6341ea6b3",
+                "sha256:ff3d0116e0ca6c096547652390025780eace3a28f6c04c9ffbf38448f1e5a87b"
             ],
             "index": "pypi",
-            "version": "==1.28.61"
+            "version": "==1.28.65"
         },
         "botocore": {
             "hashes": [
-                "sha256:39b059603f0e92a26599eecc7fe9b141f13eb412c964786ca3a7df5375928c87",
-                "sha256:433bf93af09ad205d6db4c2ffc1f0e3193ddad4e0aced0a68ad8b0fa9de903e2"
+                "sha256:90716c6f1af97e5c2f516e9a3379767ebdddcc6cbed79b026fa5038ce4e5e43e",
+                "sha256:f74e3da98dfcec17bc63ef58f82c643bf5bd7ec6cc11a26ede21cc4cd064917f"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.31.61"
+            "version": "==1.31.65"
         },
         "bravado": {
             "hashes": [
@@ -430,25 +430,25 @@
         },
         "cmake": {
             "hashes": [
-                "sha256:0ec349e938a75244ac6fbf0c5e4451ea61383d4f70173e6b09157722f4d60dde",
-                "sha256:149aaa0080f5fd1e180bed2486e602baafd8a7b1d5ac18219ff9e4378fba37e0",
-                "sha256:1edd401cc9a48562b920d3d0387130ddba4c85935f1ea777c64ba0423cb5009b",
-                "sha256:41c2aa92ce0accb43544a16e1d6bceb7f8900ac07def4ea7ba92ee95ba49be96",
-                "sha256:4c4cb267d9863281222c1f442e09c558cfac3e3454c45753ee0617692062a940",
-                "sha256:7e60dc95f78cf4ed538093d59a07744857ca344f4b1d0ef9e6508f47d0158a1d",
-                "sha256:82f6b547eb92218bb73ea5b91d7a852375d33b57bfc47f7421266ead081408e9",
-                "sha256:83f49328d37e777e711098064399d2666b243ae95d01aa275f1ad7323e973ab2",
-                "sha256:8a27928c8b387bd7df42083fe55d1490f964cb423d83861ec8d2170d2060d2c9",
-                "sha256:8ba53d322d5c84afd839138315d71ee9f892dcecb5e163e8d888f0d92cf43fe1",
-                "sha256:8c0ad28706b76d6b7c7643222d833af44bdd03da4371e3a6f0eac68e3a1d6366",
-                "sha256:94c22b97f230f4f13469215dafcf388595566a0075eaf12a527b6eb90c5c83f2",
-                "sha256:ac9b9290f67f0e7d9ccfb45396d5c6ea50dd12b46bfeb770f5a3051066fdca4e",
-                "sha256:b4f120e60823e793cc7d8645e75dce4519f76ce7692f2ab4b745e6cbca8b5345",
-                "sha256:daaa867afeeafabb1544fb129e9fbc3c537a7261430b6f00605268530f75b948",
-                "sha256:f289e26bd483e189933d1ee70987976d3fb88ce1ab6a8081c8da9d4fa44acec8",
-                "sha256:fff9dac42647c288d6c65f175de5ef0f158d3bc04eb354d6920af2c63b6d7e88"
-            ],
-            "version": "==3.27.6"
+                "sha256:009058bdf4f488709f38eaa5dd0ef0f89c6b9c6b6edd9d5b475a308ef75f80bb",
+                "sha256:1b9067ce0251cba3d4c018f2e1577ba9078e9c1eff6ad607ad5ce867843d4571",
+                "sha256:2fb48c780f1a6a3d19e785ebbb754be79d369e25a1cb81043fab049e709564da",
+                "sha256:499b38c77d52fb1964dbb38d0228fed246263a181939a8e753fde8ca227c8e1e",
+                "sha256:591f6b056527aefec009bc61a388776b2fc62444deb0038112a471031f61aeca",
+                "sha256:68983b09de633fc1ce6ab6bce9a25bfa181e41598e7c6bc0a6c0108773ee01cb",
+                "sha256:72289361866314f73be2ae63ddee224ff70223dcef9feb66d0072bf17e245564",
+                "sha256:7bf96237ba11ce2437dc5e071d96b510120a1be4708c631a64b2f38fb46bbd77",
+                "sha256:8056c99e371ff57229df2068364d7c32fea716cb53b4675f639edfb62663decf",
+                "sha256:810e592b606d05a3080a9c19ea839b13226f62cae447a22485b2365782f6b926",
+                "sha256:8bd1e1fa4fc8de7605c663d9408dceb649112f855aab05cca31fdb72e4d78364",
+                "sha256:9f4a7c7be2a25de5901f045618f41b833ea6c0f647115201d38e4fdf7e2815bc",
+                "sha256:b8a2fcb619b89d1cce7b52828316de9a1f27f0c90c2e39d1eae886428c8ee8c6",
+                "sha256:bd40d46dbad3555d5b3ce054bef24b85f256b19139493773751ab6f2b71c1219",
+                "sha256:bdbf0256f554f68c7b1d9740f5d059daf875b685c81a479cbe69038e84eb2fb9",
+                "sha256:c981aafcca2cd7210bd210ec75710c0f34e1fde1998cdcab812e4133e3ab615d",
+                "sha256:d582ef3e9ff0bd113581c1a32e881d1c2f9a34d2de76c93324a28593a76433db"
+            ],
+            "version": "==3.27.7"
         },
         "colorama": {
             "hashes": [
@@ -538,11 +538,14 @@
                 "sha256:2645f0e01735231e3c4d1568c95cc00ed477c1c2e2ed45cb123bef2ccbe72282",
                 "sha256:272ab041e95e4088321bffd63ead85d32f86f9fc79f37944663abb59e7fecbb1",
                 "sha256:2a96ff622f0db7f89d7c9aeadd9cc0c9bf61e804841a03a22ca919aa4955640d",
+                "sha256:2b5a463b01d8867f78a361defb4fadca63635965e8204f7e2409b717d85f0c1d",
+                "sha256:3cd45d09229960a89e0e3bbf71d29ac399391446f51279f3202846d7da106eb7",
                 "sha256:3d5ed0fa20b42e063ef66ad01d9948e868bbfc327bf86604e078b67f074f76f3",
                 "sha256:3f2e41dc8143d0e88ec9ba3ff66c8d2aea486b04931119abe9e394f9961d74bc",
                 "sha256:44e262f083a0d1c19bc7b8bf8aaa54d31653eab67762cdaeb68fb40b844d50b9",
                 "sha256:44fe99233ef2f42ff03d8395e7d97e0c45306eb356f6f01fa69bdb49783fdb8a",
                 "sha256:47ae97247a58d3095be800420bba7e43cc6958e67f9dfddd12decdb4c99c4d6f",
+                "sha256:4bb065d054e49645414bc84eaa49a6b7ed6c7185fb8ba2648c518344cf481144",
                 "sha256:4d56afb5f278a18743a218514825b6ab176f18a4084d8f6515c64e3acef19478",
                 "sha256:4f48e35701528beed092d97c5a1a6ba6c8902c3485d7e3a55c463162bb66afa0",
                 "sha256:53ac8df43546d3f70acbc5c17424a8c083418bd6d2cacfbd5108aaa8f3eb26db",
@@ -553,18 +556,24 @@
                 "sha256:60951e64d3e05ef2a46d2a92fc4e4563ae5e28bb3b6f231f2dca68a5078a72dc",
                 "sha256:66d6a7906fbe8c2e46e987d5a3a9cb235e931b3e4721ac7d8573151c419e3f73",
                 "sha256:6a8f0fd14bd11f2a625e5c554fd93c7775082c31ffd9fbabb4fe9db3031645d0",
+                "sha256:6bf4ec082a7647ed7463c5e0bd176850a9f013c258ad98f53055bcd1716c7fac",
                 "sha256:71bf6a6632648333c402a8692fe61f45416066eb0d8b7f4530cdf37fee221a11",
                 "sha256:79c36d95e89b43c29595c889c7a4d30d29aefc55d7c58a26a058b9bbe7abd5cf",
                 "sha256:7b0fa1f8600fdf093a4e5eac5dc56084047abec50d589a8121618e3fb1f9de3f",
                 "sha256:7e44dda432a8c8a47cb73869201d2f9777604bd913d859def84c659fb736cfd3",
                 "sha256:820421600d395d0e11b9da1e983acde453637090f5a15abb45f5182cd35f9c3a",
+                "sha256:831951a3eccebd5911387e3fff14483747d3dae5ad496afbd50447ffe2d03dba",
                 "sha256:89b683919ffc846b91b405d5f14df8a2ba1167f4ed9277150298fbe91a3d4aae",
+                "sha256:8e82276b120000834af3f776cbe22ac3016cd9e0ed00e7f3fb31e2ce95e17181",
                 "sha256:8ea1c781d3760df0b6ad80b7b19dc8e038e0638fb1cfabc68da96cedb8d0adca",
                 "sha256:91bb03ca0d3857f319afd13525d5c6214a55aa49778ce46a02c16f0eee37907c",
                 "sha256:9698ed7fe78b4de81dcee0624311ee93a4436f0b3e3c6b147a6745a3fd210272",
+                "sha256:9a3f45cd17730c00358643a814d5921a31c8a0d238582e08594fa86fdbc8401c",
+                "sha256:9f9e403be6307e98a47cc7d0132fe1fe4683c873c5d7d9da099fbac9c299a4b0",
                 "sha256:a08dcb7c7b54f82db4ee9120aaace06326499c0d4108770ee7ac63d7bd1d803d",
                 "sha256:a7f93316abc1abfd348b04afc6cadbbd4fba44cd91e7b9803c9330045a7a1885",
                 "sha256:aac9d49e16f473ceb4eaf74a53180eac3363127f01855c39122b400a988e80bf",
+                "sha256:b84565611f36ee8c3944b96bb01ee4d44c101acf84f1430c9414986ab3a7fb03",
                 "sha256:bee04fd1cdd5f2a2e91e4b271f22e228c698fe7b7f8ef209374d717f7889e80c",
                 "sha256:bef07e7d4607c4d70627e58eb630fe60e48b80a61ab05b33314e3296eb90af78",
                 "sha256:c2602c42ed101ada634fa37253d40946f0468b2b749689781cba715a7d78038e",
@@ -584,7 +593,8 @@
                 "sha256:f9340c70d95ea102cf51a07ecc09f93f1363e585d97b5276734a5f8c4476e560",
                 "sha256:fab800ed93fb129d85c63f26660b695fb194efb29765a163f269321778e28a8d",
                 "sha256:fbc59b223750a901d65d96333461ab17076594fa34448ed2ef911bd4b628f068",
-                "sha256:fbffb1f63edf4cb4272a25de288c2f2e20914bb93e003883656774e61794b960"
+                "sha256:fbffb1f63edf4cb4272a25de288c2f2e20914bb93e003883656774e61794b960",
+                "sha256:fd2e81c69baacb95fa28cdf844f714d7c03f0c805f4fa2decc5e9565e6b4405d"
             ],
             "markers": "python_version >= '3.7'",
             "version": "==2.7.0"
@@ -626,6 +636,13 @@
             "index": "pypi",
             "version": "==2.14.5"
         },
+        "deepspeed": {
+            "hashes": [
+                "sha256:fa342f11f971e1bb002a537adfea6ac4c089b7a4da5aef5f2216a472cb35be7f"
+            ],
+            "index": "pypi",
+            "version": "==0.11.1"
+        },
         "dill": {
             "hashes": [
                 "sha256:76b122c08ef4ce2eedcd4d1abd8e641114bfc6c2867f49f3c41facf65bf19f5e",
@@ -786,7 +803,7 @@
             "hashes": [
                 "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307"
             ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==0.18.3"
         },
         "gitdb": {
@@ -799,11 +816,11 @@
         },
         "gitpython": {
             "hashes": [
-                "sha256:5f4c4187de49616d710a77e98ddf17b4782060a1788df441846bddefbb89ab33",
-                "sha256:f9b9ddc0761c125d5780eab2d64be4873fc6817c2899cbcb34b02344bdc7bc54"
+                "sha256:22b126e9ffb671fdd0c129796343a02bf67bf2994b35449ffc9321aa755e18a4",
+                "sha256:cf14627d5a8049ffbf49915732e5eddbe8134c3bdb9d476e6182b676fc573f8a"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.1.37"
+            "version": "==3.1.40"
         },
         "gputil": {
             "hashes": [
@@ -877,7 +894,7 @@
                 "sha256:f351479a6914fd81a55c8e68963609f792d9b067fb8a60a042c585a621e0de4f",
                 "sha256:f47932c434a3c8d3c86d865443fadc1fbf574e9b11d6650b656e602b1797908a"
             ],
-            "markers": "python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))",
+            "markers": "python_version >= '3' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))))",
             "version": "==3.0.0"
         },
         "h11": {
@@ -925,6 +942,13 @@
             "index": "pypi",
             "version": "==0.1.3"
         },
+        "hjson": {
+            "hashes": [
+                "sha256:55af475a27cf83a7969c808399d7bccdec8fb836a07ddbd574587593b9cdcf75",
+                "sha256:65713cdcf13214fb554eb8b4ef803419733f4f5e551047c9b711098ab7186b89"
+            ],
+            "version": "==3.1.0"
+        },
         "httpcore": {
             "hashes": [
                 "sha256:13b5e5cd1dca1a6636a6aaea212b19f4f85cd88c366a2b82304181b769aab3c9",
@@ -1075,9 +1099,9 @@
         },
         "lit": {
             "hashes": [
-                "sha256:d6a551eab550f81023c82a260cd484d63970d2be9fd7588111208e7d2ff62212"
+                "sha256:e6049032462be1e2928686cbd4a6cc5b3c545d83ecd078737fe79412c1f3fcc1"
             ],
-            "version": "==17.0.2"
+            "version": "==17.0.3"
         },
         "markupsafe": {
             "hashes": [
@@ -1347,43 +1371,63 @@
             "markers": "python_version >= '3.8'",
             "version": "==3.1"
         },
+        "ninja": {
+            "hashes": [
+                "sha256:18302d96a5467ea98b68e1cae1ae4b4fb2b2a56a82b955193c637557c7273dbd",
+                "sha256:185e0641bde601e53841525c4196278e9aaf4463758da6dd1e752c0a0f54136a",
+                "sha256:376889c76d87b95b5719fdd61dd7db193aa7fd4432e5d52d2e44e4c497bdbbee",
+                "sha256:3e0f9be5bb20d74d58c66cc1c414c3e6aeb45c35b0d0e41e8d739c2c0d57784f",
+                "sha256:73b93c14046447c7c5cc892433d4fae65d6364bec6685411cb97a8bcf815f93a",
+                "sha256:7563ce1d9fe6ed5af0b8dd9ab4a214bf4ff1f2f6fd6dc29f480981f0f8b8b249",
+                "sha256:76482ba746a2618eecf89d5253c0d1e4f1da1270d41e9f54dfbd91831b0f6885",
+                "sha256:84502ec98f02a037a169c4b0d5d86075eaf6afc55e1879003d6cab51ced2ea4b",
+                "sha256:95da904130bfa02ea74ff9c0116b4ad266174fafb1c707aa50212bc7859aebf1",
+                "sha256:9d793b08dd857e38d0b6ffe9e6b7145d7c485a42dcfea04905ca0cdb6017cc3c",
+                "sha256:9df724344202b83018abb45cb1efc22efd337a1496514e7e6b3b59655be85205",
+                "sha256:aad34a70ef15b12519946c5633344bc775a7656d789d9ed5fdb0d456383716ef",
+                "sha256:d491fc8d89cdcb416107c349ad1e3a735d4c4af5e1cb8f5f727baca6350fdaea",
+                "sha256:ecf80cf5afd09f14dcceff28cb3f11dc90fb97c999c89307aea435889cb66877",
+                "sha256:fa2ba9d74acfdfbfbcf06fad1b8282de8a7a8c481d9dee45c859a8c93fcc1082"
+            ],
+            "version": "==1.11.1.1"
+        },
         "numpy": {
             "hashes": [
-                "sha256:020cdbee66ed46b671429c7265cf00d8ac91c046901c55684954c3958525dab2",
-                "sha256:0621f7daf973d34d18b4e4bafb210bbaf1ef5e0100b5fa750bd9cde84c7ac292",
-                "sha256:0792824ce2f7ea0c82ed2e4fecc29bb86bee0567a080dacaf2e0a01fe7654369",
-                "sha256:09aaee96c2cbdea95de76ecb8a586cb687d281c881f5f17bfc0fb7f5890f6b91",
-                "sha256:166b36197e9debc4e384e9c652ba60c0bacc216d0fc89e78f973a9760b503388",
-                "sha256:186ba67fad3c60dbe8a3abff3b67a91351100f2661c8e2a80364ae6279720299",
-                "sha256:306545e234503a24fe9ae95ebf84d25cba1fdc27db971aa2d9f1ab6bba19a9dd",
-                "sha256:436c8e9a4bdeeee84e3e59614d38c3dbd3235838a877af8c211cfcac8a80b8d3",
-                "sha256:4a873a8180479bc829313e8d9798d5234dfacfc2e8a7ac188418189bb8eafbd2",
-                "sha256:4acc65dd65da28060e206c8f27a573455ed724e6179941edb19f97e58161bb69",
-                "sha256:51be5f8c349fdd1a5568e72713a21f518e7d6707bcf8503b528b88d33b57dc68",
-                "sha256:546b7dd7e22f3c6861463bebb000646fa730e55df5ee4a0224408b5694cc6148",
-                "sha256:5671338034b820c8d58c81ad1dafc0ed5a00771a82fccc71d6438df00302094b",
-                "sha256:637c58b468a69869258b8ae26f4a4c6ff8abffd4a8334c830ffb63e0feefe99a",
-                "sha256:767254ad364991ccfc4d81b8152912e53e103ec192d1bb4ea6b1f5a7117040be",
-                "sha256:7d484292eaeb3e84a51432a94f53578689ffdea3f90e10c8b203a99be5af57d8",
-                "sha256:7f6bad22a791226d0a5c7c27a80a20e11cfe09ad5ef9084d4d3fc4a299cca505",
-                "sha256:86f737708b366c36b76e953c46ba5827d8c27b7a8c9d0f471810728e5a2fe57c",
-                "sha256:8c6adc33561bd1d46f81131d5352348350fc23df4d742bb246cdfca606ea1208",
-                "sha256:914b28d3215e0c721dc75db3ad6d62f51f630cb0c277e6b3bcb39519bed10bd8",
-                "sha256:b44e6a09afc12952a7d2a58ca0a2429ee0d49a4f89d83a0a11052da696440e49",
-                "sha256:bb0d9a1aaf5f1cb7967320e80690a1d7ff69f1d47ebc5a9bea013e3a21faec95",
-                "sha256:c0b45c8b65b79337dee5134d038346d30e109e9e2e9d43464a2970e5c0e93229",
-                "sha256:c2e698cb0c6dda9372ea98a0344245ee65bdc1c9dd939cceed6bb91256837896",
-                "sha256:c78a22e95182fb2e7874712433eaa610478a3caf86f28c621708d35fa4fd6e7f",
-                "sha256:e062aa24638bb5018b7841977c360d2f5917268d125c833a686b7cbabbec496c",
-                "sha256:e5e18e5b14a7560d8acf1c596688f4dfd19b4f2945b245a71e5af4ddb7422feb",
-                "sha256:eae430ecf5794cb7ae7fa3808740b015aa80747e5266153128ef055975a72b99",
-                "sha256:ee84ca3c58fe48b8ddafdeb1db87388dce2c3c3f701bf447b05e4cfcc3679112",
-                "sha256:f042f66d0b4ae6d48e70e28d487376204d3cbf43b84c03bac57e28dac6151581",
-                "sha256:f8db2f125746e44dce707dd44d4f4efeea8d7e2b43aace3f8d1f235cfa2733dd",
-                "sha256:f93fc78fe8bf15afe2b8d6b6499f1c73953169fad1e9a8dd086cdff3190e7fdf"
+                "sha256:06934e1a22c54636a059215d6da99e23286424f316fddd979f5071093b648668",
+                "sha256:1c59c046c31a43310ad0199d6299e59f57a289e22f0f36951ced1c9eac3665b9",
+                "sha256:1d1bd82d539607951cac963388534da3b7ea0e18b149a53cf883d8f699178c0f",
+                "sha256:1e11668d6f756ca5ef534b5be8653d16c5352cbb210a5c2a79ff288e937010d5",
+                "sha256:3649d566e2fc067597125428db15d60eb42a4e0897fc48d28cb75dc2e0454e53",
+                "sha256:59227c981d43425ca5e5c01094d59eb14e8772ce6975d4b2fc1e106a833d5ae2",
+                "sha256:6081aed64714a18c72b168a9276095ef9155dd7888b9e74b5987808f0dd0a974",
+                "sha256:6965888d65d2848e8768824ca8288db0a81263c1efccec881cb35a0d805fcd2f",
+                "sha256:76ff661a867d9272cd2a99eed002470f46dbe0943a5ffd140f49be84f68ffc42",
+                "sha256:78ca54b2f9daffa5f323f34cdf21e1d9779a54073f0018a3094ab907938331a2",
+                "sha256:82e871307a6331b5f09efda3c22e03c095d957f04bf6bc1804f30048d0e5e7af",
+                "sha256:8ab9163ca8aeb7fd32fe93866490654d2f7dda4e61bc6297bf72ce07fdc02f67",
+                "sha256:9696aa2e35cc41e398a6d42d147cf326f8f9d81befcb399bc1ed7ffea339b64e",
+                "sha256:97e5d6a9f0702c2863aaabf19f0d1b6c2628fbe476438ce0b5ce06e83085064c",
+                "sha256:9f42284ebf91bdf32fafac29d29d4c07e5e9d1af862ea73686581773ef9e73a7",
+                "sha256:a03fb25610ef560a6201ff06df4f8105292ba56e7cdd196ea350d123fc32e24e",
+                "sha256:a5b411040beead47a228bde3b2241100454a6abde9df139ed087bd73fc0a4908",
+                "sha256:af22f3d8e228d84d1c0c44c1fbdeb80f97a15a0abe4f080960393a00db733b66",
+                "sha256:afd5ced4e5a96dac6725daeb5242a35494243f2239244fad10a90ce58b071d24",
+                "sha256:b9d45d1dbb9de84894cc50efece5b09939752a2d75aab3a8b0cef6f3a35ecd6b",
+                "sha256:bb894accfd16b867d8643fc2ba6c8617c78ba2828051e9a69511644ce86ce83e",
+                "sha256:c8c6c72d4a9f831f328efb1312642a1cafafaa88981d9ab76368d50d07d93cbe",
+                "sha256:cd7837b2b734ca72959a1caf3309457a318c934abef7a43a14bb984e574bbb9a",
+                "sha256:cdd9ec98f0063d93baeb01aad472a1a0840dee302842a2746a7a8e92968f9575",
+                "sha256:d1cfc92db6af1fd37a7bb58e55c8383b4aa1ba23d012bdbba26b4bcca45ac297",
+                "sha256:d1d2c6b7dd618c41e202c59c1413ef9b2c8e8a15f5039e344af64195459e3104",
+                "sha256:d2984cb6caaf05294b8466966627e80bf6c7afd273279077679cb010acb0e5ab",
+                "sha256:d58e8c51a7cf43090d124d5073bc29ab2755822181fcad978b12e144e5e5a4b3",
+                "sha256:d78f269e0c4fd365fc2992c00353e4530d274ba68f15e968d8bc3c69ce5f5244",
+                "sha256:dcfaf015b79d1f9f9c9fd0731a907407dc3e45769262d657d754c3a028586124",
+                "sha256:e44ccb93f30c75dfc0c3aa3ce38f33486a75ec9abadabd4e59f114994a9c4617",
+                "sha256:e509cbc488c735b43b5ffea175235cec24bbc57b227ef1acc691725beb230d1c"
             ],
             "index": "pypi",
-            "version": "==1.26.0"
+            "version": "==1.26.1"
         },
         "oauthlib": {
             "hashes": [
@@ -1450,63 +1494,63 @@
         },
         "pillow": {
             "hashes": [
-                "sha256:0462b1496505a3462d0f35dc1c4d7b54069747d65d00ef48e736acda2c8cbdff",
-                "sha256:186f7e04248103482ea6354af6d5bcedb62941ee08f7f788a1c7707bc720c66f",
-                "sha256:19e9adb3f22d4c416e7cd79b01375b17159d6990003633ff1d8377e21b7f1b21",
-                "sha256:28444cb6ad49726127d6b340217f0627abc8732f1194fd5352dec5e6a0105635",
-                "sha256:2872f2d7846cf39b3dbff64bc1104cc48c76145854256451d33c5faa55c04d1a",
-                "sha256:2cc6b86ece42a11f16f55fe8903595eff2b25e0358dec635d0a701ac9586588f",
-                "sha256:2d7e91b4379f7a76b31c2dda84ab9e20c6220488e50f7822e59dac36b0cd92b1",
-                "sha256:2fa6dd2661838c66f1a5473f3b49ab610c98a128fc08afbe81b91a1f0bf8c51d",
-                "sha256:32bec7423cdf25c9038fef614a853c9d25c07590e1a870ed471f47fb80b244db",
-                "sha256:3855447d98cced8670aaa63683808df905e956f00348732448b5a6df67ee5849",
-                "sha256:3a04359f308ebee571a3127fdb1bd01f88ba6f6fb6d087f8dd2e0d9bff43f2a7",
-                "sha256:3a0d3e54ab1df9df51b914b2233cf779a5a10dfd1ce339d0421748232cea9876",
-                "sha256:44e7e4587392953e5e251190a964675f61e4dae88d1e6edbe9f36d6243547ff3",
-                "sha256:459307cacdd4138edee3875bbe22a2492519e060660eaf378ba3b405d1c66317",
-                "sha256:4ce90f8a24e1c15465048959f1e94309dfef93af272633e8f37361b824532e91",
-                "sha256:50bd5f1ebafe9362ad622072a1d2f5850ecfa44303531ff14353a4059113b12d",
-                "sha256:522ff4ac3aaf839242c6f4e5b406634bfea002469656ae8358644fc6c4856a3b",
-                "sha256:552912dbca585b74d75279a7570dd29fa43b6d93594abb494ebb31ac19ace6bd",
-                "sha256:5d6c9049c6274c1bb565021367431ad04481ebb54872edecfcd6088d27edd6ed",
-                "sha256:697a06bdcedd473b35e50a7e7506b1d8ceb832dc238a336bd6f4f5aa91a4b500",
-                "sha256:71671503e3015da1b50bd18951e2f9daf5b6ffe36d16f1eb2c45711a301521a7",
-                "sha256:723bd25051454cea9990203405fa6b74e043ea76d4968166dfd2569b0210886a",
-                "sha256:764d2c0daf9c4d40ad12fbc0abd5da3af7f8aa11daf87e4fa1b834000f4b6b0a",
-                "sha256:787bb0169d2385a798888e1122c980c6eff26bf941a8ea79747d35d8f9210ca0",
-                "sha256:7f771e7219ff04b79e231d099c0a28ed83aa82af91fd5fa9fdb28f5b8d5addaf",
-                "sha256:847e8d1017c741c735d3cd1883fa7b03ded4f825a6e5fcb9378fd813edee995f",
-                "sha256:84efb46e8d881bb06b35d1d541aa87f574b58e87f781cbba8d200daa835b42e1",
-                "sha256:898f1d306298ff40dc1b9ca24824f0488f6f039bc0e25cfb549d3195ffa17088",
-                "sha256:8b451d6ead6e3500b6ce5c7916a43d8d8d25ad74b9102a629baccc0808c54971",
-                "sha256:8f06be50669087250f319b706decf69ca71fdecd829091a37cc89398ca4dc17a",
-                "sha256:92a23b0431941a33242b1f0ce6c88a952e09feeea9af4e8be48236a68ffe2205",
-                "sha256:93139acd8109edcdeffd85e3af8ae7d88b258b3a1e13a038f542b79b6d255c54",
-                "sha256:98533fd7fa764e5f85eebe56c8e4094db912ccbe6fbf3a58778d543cadd0db08",
-                "sha256:9f665d1e6474af9f9da5e86c2a3a2d2d6204e04d5af9c06b9d42afa6ebde3f21",
-                "sha256:b059ac2c4c7a97daafa7dc850b43b2d3667def858a4f112d1aa082e5c3d6cf7d",
-                "sha256:b1be1c872b9b5fcc229adeadbeb51422a9633abd847c0ff87dc4ef9bb184ae08",
-                "sha256:b7cf63d2c6928b51d35dfdbda6f2c1fddbe51a6bc4a9d4ee6ea0e11670dd981e",
-                "sha256:bc2e3069569ea9dbe88d6b8ea38f439a6aad8f6e7a6283a38edf61ddefb3a9bf",
-                "sha256:bcf1207e2f2385a576832af02702de104be71301c2696d0012b1b93fe34aaa5b",
-                "sha256:ca26ba5767888c84bf5a0c1a32f069e8204ce8c21d00a49c90dabeba00ce0145",
-                "sha256:cbe68deb8580462ca0d9eb56a81912f59eb4542e1ef8f987405e35a0179f4ea2",
-                "sha256:d6caf3cd38449ec3cd8a68b375e0c6fe4b6fd04edb6c9766b55ef84a6e8ddf2d",
-                "sha256:d72967b06be9300fed5cfbc8b5bafceec48bf7cdc7dab66b1d2549035287191d",
-                "sha256:d889b53ae2f030f756e61a7bff13684dcd77e9af8b10c6048fb2c559d6ed6eaf",
-                "sha256:de596695a75496deb3b499c8c4f8e60376e0516e1a774e7bc046f0f48cd620ad",
-                "sha256:e6a90167bcca1216606223a05e2cf991bb25b14695c518bc65639463d7db722d",
-                "sha256:ed2d9c0704f2dc4fa980b99d565c0c9a543fe5101c25b3d60488b8ba80f0cce1",
-                "sha256:ee7810cf7c83fa227ba9125de6084e5e8b08c59038a7b2c9045ef4dde61663b4",
-                "sha256:f0b4b06da13275bc02adfeb82643c4a6385bd08d26f03068c2796f60d125f6f2",
-                "sha256:f11c9102c56ffb9ca87134bd025a43d2aba3f1155f508eff88f694b33a9c6d19",
-                "sha256:f5bb289bb835f9fe1a1e9300d011eef4d69661bb9b34d5e196e5e82c4cb09b37",
-                "sha256:f6d3d4c905e26354e8f9d82548475c46d8e0889538cb0657aa9c6f0872a37aa4",
-                "sha256:fcb59711009b0168d6ee0bd8fb5eb259c4ab1717b2f538bbf36bacf207ef7a68",
-                "sha256:fd2a5403a75b54661182b75ec6132437a181209b901446ee5724b589af8edef1"
+                "sha256:00f438bb841382b15d7deb9a05cc946ee0f2c352653c7aa659e75e592f6fa17d",
+                "sha256:0248f86b3ea061e67817c47ecbe82c23f9dd5d5226200eb9090b3873d3ca32de",
+                "sha256:04f6f6149f266a100374ca3cc368b67fb27c4af9f1cc8cb6306d849dcdf12616",
+                "sha256:062a1610e3bc258bff2328ec43f34244fcec972ee0717200cb1425214fe5b839",
+                "sha256:0a026c188be3b443916179f5d04548092e253beb0c3e2ee0a4e2cdad72f66099",
+                "sha256:0f7c276c05a9767e877a0b4c5050c8bee6a6d960d7f0c11ebda6b99746068c2a",
+                "sha256:1a8413794b4ad9719346cd9306118450b7b00d9a15846451549314a58ac42219",
+                "sha256:1ab05f3db77e98f93964697c8efc49c7954b08dd61cff526b7f2531a22410106",
+                "sha256:1c3ac5423c8c1da5928aa12c6e258921956757d976405e9467c5f39d1d577a4b",
+                "sha256:1c41d960babf951e01a49c9746f92c5a7e0d939d1652d7ba30f6b3090f27e412",
+                "sha256:1fafabe50a6977ac70dfe829b2d5735fd54e190ab55259ec8aea4aaea412fa0b",
+                "sha256:1fb29c07478e6c06a46b867e43b0bcdb241b44cc52be9bc25ce5944eed4648e7",
+                "sha256:24fadc71218ad2b8ffe437b54876c9382b4a29e030a05a9879f615091f42ffc2",
+                "sha256:2cdc65a46e74514ce742c2013cd4a2d12e8553e3a2563c64879f7c7e4d28bce7",
+                "sha256:2ef6721c97894a7aa77723740a09547197533146fba8355e86d6d9a4a1056b14",
+                "sha256:3b834f4b16173e5b92ab6566f0473bfb09f939ba14b23b8da1f54fa63e4b623f",
+                "sha256:3d929a19f5469b3f4df33a3df2983db070ebb2088a1e145e18facbc28cae5b27",
+                "sha256:41f67248d92a5e0a2076d3517d8d4b1e41a97e2df10eb8f93106c89107f38b57",
+                "sha256:47e5bf85b80abc03be7455c95b6d6e4896a62f6541c1f2ce77a7d2bb832af262",
+                "sha256:4d0152565c6aa6ebbfb1e5d8624140a440f2b99bf7afaafbdbf6430426497f28",
+                "sha256:50d08cd0a2ecd2a8657bd3d82c71efd5a58edb04d9308185d66c3a5a5bed9610",
+                "sha256:61f1a9d247317fa08a308daaa8ee7b3f760ab1809ca2da14ecc88ae4257d6172",
+                "sha256:6932a7652464746fcb484f7fc3618e6503d2066d853f68a4bd97193a3996e273",
+                "sha256:7a7e3daa202beb61821c06d2517428e8e7c1aab08943e92ec9e5755c2fc9ba5e",
+                "sha256:7dbaa3c7de82ef37e7708521be41db5565004258ca76945ad74a8e998c30af8d",
+                "sha256:7df5608bc38bd37ef585ae9c38c9cd46d7c81498f086915b0f97255ea60c2818",
+                "sha256:806abdd8249ba3953c33742506fe414880bad78ac25cc9a9b1c6ae97bedd573f",
+                "sha256:883f216eac8712b83a63f41b76ddfb7b2afab1b74abbb413c5df6680f071a6b9",
+                "sha256:912e3812a1dbbc834da2b32299b124b5ddcb664ed354916fd1ed6f193f0e2d01",
+                "sha256:937bdc5a7f5343d1c97dc98149a0be7eb9704e937fe3dc7140e229ae4fc572a7",
+                "sha256:9882a7451c680c12f232a422730f986a1fcd808da0fd428f08b671237237d651",
+                "sha256:9a92109192b360634a4489c0c756364c0c3a2992906752165ecb50544c251312",
+                "sha256:9d7bc666bd8c5a4225e7ac71f2f9d12466ec555e89092728ea0f5c0c2422ea80",
+                "sha256:a5f63b5a68daedc54c7c3464508d8c12075e56dcfbd42f8c1bf40169061ae666",
+                "sha256:a646e48de237d860c36e0db37ecaecaa3619e6f3e9d5319e527ccbc8151df061",
+                "sha256:a89b8312d51715b510a4fe9fc13686283f376cfd5abca8cd1c65e4c76e21081b",
+                "sha256:a92386125e9ee90381c3369f57a2a50fa9e6aa8b1cf1d9c4b200d41a7dd8e992",
+                "sha256:ae88931f93214777c7a3aa0a8f92a683f83ecde27f65a45f95f22d289a69e593",
+                "sha256:afc8eef765d948543a4775f00b7b8c079b3321d6b675dde0d02afa2ee23000b4",
+                "sha256:b0eb01ca85b2361b09480784a7931fc648ed8b7836f01fb9241141b968feb1db",
+                "sha256:b1c25762197144e211efb5f4e8ad656f36c8d214d390585d1d21281f46d556ba",
+                "sha256:b4005fee46ed9be0b8fb42be0c20e79411533d1fd58edabebc0dd24626882cfd",
+                "sha256:b920e4d028f6442bea9a75b7491c063f0b9a3972520731ed26c83e254302eb1e",
+                "sha256:baada14941c83079bf84c037e2d8b7506ce201e92e3d2fa0d1303507a8538212",
+                "sha256:bb40c011447712d2e19cc261c82655f75f32cb724788df315ed992a4d65696bb",
+                "sha256:c0949b55eb607898e28eaccb525ab104b2d86542a85c74baf3a6dc24002edec2",
+                "sha256:c9aeea7b63edb7884b031a35305629a7593272b54f429a9869a4f63a1bf04c34",
+                "sha256:cfe96560c6ce2f4c07d6647af2d0f3c54cc33289894ebd88cfbb3bcd5391e256",
+                "sha256:d27b5997bdd2eb9fb199982bb7eb6164db0426904020dc38c10203187ae2ff2f",
+                "sha256:d921bc90b1defa55c9917ca6b6b71430e4286fc9e44c55ead78ca1a9f9eba5f2",
+                "sha256:e6bf8de6c36ed96c86ea3b6e1d5273c53f46ef518a062464cd7ef5dd2cf92e38",
+                "sha256:eaed6977fa73408b7b8a24e8b14e59e1668cfc0f4c40193ea7ced8e210adf996",
+                "sha256:fa1d323703cfdac2036af05191b969b910d8f115cf53093125e4058f62012c9a",
+                "sha256:fe1e26e1ffc38be097f0ba1d0d07fcade2bcfd1d023cda5b29935ae8052bd793"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==10.0.1"
+            "version": "==10.1.0"
         },
         "portalocker": {
             "hashes": [
@@ -1546,23 +1590,32 @@
         },
         "psutil": {
             "hashes": [
-                "sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d",
-                "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217",
-                "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4",
-                "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c",
-                "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f",
-                "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da",
-                "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4",
-                "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42",
-                "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5",
-                "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4",
-                "sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9",
-                "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f",
-                "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30",
-                "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"
+                "sha256:10e8c17b4f898d64b121149afb136c53ea8b68c7531155147867b7b1ac9e7e28",
+                "sha256:18cd22c5db486f33998f37e2bb054cc62fd06646995285e02a51b1e08da97017",
+                "sha256:3ebf2158c16cc69db777e3c7decb3c0f43a7af94a60d72e87b2823aebac3d602",
+                "sha256:51dc3d54607c73148f63732c727856f5febec1c7c336f8f41fcbd6315cce76ac",
+                "sha256:6e5fb8dc711a514da83098bc5234264e551ad980cec5f85dabf4d38ed6f15e9a",
+                "sha256:70cb3beb98bc3fd5ac9ac617a327af7e7f826373ee64c80efd4eb2856e5051e9",
+                "sha256:748c9dd2583ed86347ed65d0035f45fa8c851e8d90354c122ab72319b5f366f4",
+                "sha256:91ecd2d9c00db9817a4b4192107cf6954addb5d9d67a969a4f436dbc9200f88c",
+                "sha256:92e0cc43c524834af53e9d3369245e6cc3b130e78e26100d1f63cdb0abeb3d3c",
+                "sha256:a6f01f03bf1843280f4ad16f4bde26b817847b4c1a0db59bf6419807bc5ce05c",
+                "sha256:c69596f9fc2f8acd574a12d5f8b7b1ba3765a641ea5d60fb4736bf3c08a8214a",
+                "sha256:ca2780f5e038379e520281e4c032dddd086906ddff9ef0d1b9dcf00710e5071c",
+                "sha256:daecbcbd29b289aac14ece28eca6a3e60aa361754cf6da3dfb20d4d32b6c7f57",
+                "sha256:e4b92ddcd7dd4cdd3f900180ea1e104932c7bce234fb88976e2a3b296441225a",
+                "sha256:fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d",
+                "sha256:ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==5.9.5"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+            "version": "==5.9.6"
+        },
+        "py-cpuinfo": {
+            "hashes": [
+                "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690",
+                "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5"
+            ],
+            "version": "==9.0.0"
         },
         "pyarrow": {
             "hashes": [
@@ -1606,6 +1659,48 @@
             ],
             "version": "==2.21"
         },
+        "pydantic": {
+            "hashes": [
+                "sha256:1740068fd8e2ef6eb27a20e5651df000978edce6da6803c2bef0bc74540f9548",
+                "sha256:210ce042e8f6f7c01168b2d84d4c9eb2b009fe7bf572c2266e235edf14bacd80",
+                "sha256:32c8b48dcd3b2ac4e78b0ba4af3a2c2eb6048cb75202f0ea7b34feb740efc340",
+                "sha256:3ecea2b9d80e5333303eeb77e180b90e95eea8f765d08c3d278cd56b00345d01",
+                "sha256:4b03e42ec20286f052490423682016fd80fda830d8e4119f8ab13ec7464c0132",
+                "sha256:4c5370a7edaac06daee3af1c8b1192e305bc102abcbf2a92374b5bc793818599",
+                "sha256:56e3ff861c3b9c6857579de282ce8baabf443f42ffba355bf070770ed63e11e1",
+                "sha256:5a1f9f747851338933942db7af7b6ee8268568ef2ed86c4185c6ef4402e80ba8",
+                "sha256:5e08865bc6464df8c7d61439ef4439829e3ab62ab1669cddea8dd00cd74b9ffe",
+                "sha256:61d9dce220447fb74f45e73d7ff3b530e25db30192ad8d425166d43c5deb6df0",
+                "sha256:654db58ae399fe6434e55325a2c3e959836bd17a6f6a0b6ca8107ea0571d2e17",
+                "sha256:678bcf5591b63cc917100dc50ab6caebe597ac67e8c9ccb75e698f66038ea953",
+                "sha256:6cf25c1a65c27923a17b3da28a0bdb99f62ee04230c931d83e888012851f4e7f",
+                "sha256:75ac15385a3534d887a99c713aa3da88a30fbd6204a5cd0dc4dab3d770b9bd2f",
+                "sha256:75b297827b59bc229cac1a23a2f7a4ac0031068e5be0ce385be1462e7e17a35d",
+                "sha256:7d6f6e7305244bddb4414ba7094ce910560c907bdfa3501e9db1a7fd7eaea127",
+                "sha256:84bafe2e60b5e78bc64a2941b4c071a4b7404c5c907f5f5a99b0139781e69ed8",
+                "sha256:854223752ba81e3abf663d685f105c64150873cc6f5d0c01d3e3220bcff7d36f",
+                "sha256:8ae5dd6b721459bfa30805f4c25880e0dd78fc5b5879f9f7a692196ddcb5a580",
+                "sha256:8ef467901d7a41fa0ca6db9ae3ec0021e3f657ce2c208e98cd511f3161c762c6",
+                "sha256:968ac42970f57b8344ee08837b62f6ee6f53c33f603547a55571c954a4225691",
+                "sha256:97cce3ae7341f7620a0ba5ef6cf043975cd9d2b81f3aa5f4ea37928269bc1b87",
+                "sha256:9849f031cf8a2f0a928fe885e5a04b08006d6d41876b8bbd2fc68a18f9f2e3fd",
+                "sha256:9f00790179497767aae6bcdc36355792c79e7bbb20b145ff449700eb076c5f96",
+                "sha256:b87326822e71bd5f313e7d3bfdc77ac3247035ac10b0c0618bd99dcf95b1e687",
+                "sha256:b97c1fac8c49be29486df85968682b0afa77e1b809aff74b83081cc115e52f33",
+                "sha256:bc0898c12f8e9c97f6cd44c0ed70d55749eaf783716896960b4ecce2edfd2d69",
+                "sha256:c553f6a156deb868ba38a23cf0df886c63492e9257f60a79c0fd8e7173537653",
+                "sha256:c636925f38b8db208e09d344c7aa4f29a86bb9947495dd6b6d376ad10334fb78",
+                "sha256:c958d053453a1c4b1c2062b05cd42d9d5c8eb67537b8d5a7e3c3032943ecd261",
+                "sha256:d3a3c792a58e1622667a2837512099eac62490cdfd63bd407993aaf200a4cf1f",
+                "sha256:e31647d85a2013d926ce60b84f9dd5300d44535a9941fe825dc349ae1f760df9",
+                "sha256:e70ca129d2053fb8b728ee7d1af8e553a928d7e301a311094b8a0501adc8763d",
+                "sha256:efff03cc7a4f29d9009d1c96ceb1e7a70a65cfe86e89d34e4a5f2ab1e5693737",
+                "sha256:f59ef915cac80275245824e9d771ee939133be38215555e9dc90c6cb148aaeb5",
+                "sha256:f8e81fc5fb17dae698f52bdd1c4f18b6ca674d7068242b2aff075f588301bbb0"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.10.13"
+        },
         "pyjwt": {
             "hashes": [
                 "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de",
@@ -1619,7 +1714,7 @@
                 "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
                 "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==2.8.2"
         },
         "python-editor": {
@@ -1845,108 +1940,108 @@
         },
         "rpds-py": {
             "hashes": [
-                "sha256:00a88003db3cc953f8656b59fc9af9d0637a1fb93c235814007988f8c153b2f2",
-                "sha256:049098dabfe705e9638c55a3321137a821399c50940041a6fcce267a22c70db2",
-                "sha256:08f07150c8ebbdbce1d2d51b8e9f4d588749a2af6a98035485ebe45c7ad9394e",
-                "sha256:125776d5db15162fdd9135372bef7fe4fb7c5f5810cf25898eb74a06a0816aec",
-                "sha256:13cbd79ccedc6b39c279af31ebfb0aec0467ad5d14641ddb15738bf6e4146157",
-                "sha256:18d5ff7fbd305a1d564273e9eb22de83ae3cd9cd6329fddc8f12f6428a711a6a",
-                "sha256:1c27942722cd5039bbf5098c7e21935a96243fed00ea11a9589f3c6c6424bd84",
-                "sha256:255a23bded80605e9f3997753e3a4b89c9aec9efb07ec036b1ca81440efcc1a9",
-                "sha256:2573ec23ad3a59dd2bc622befac845695972f3f2d08dc1a4405d017d20a6c225",
-                "sha256:2603e084054351cc65097da326570102c4c5bd07426ba8471ceaefdb0b642cc9",
-                "sha256:28b4942ec7d9d6114c1e08cace0157db92ef674636a38093cab779ace5742d3a",
-                "sha256:28e29dac59df890972f73c511948072897f512974714a803fe793635b80ff8c7",
-                "sha256:2a97406d5e08b7095428f01dac0d3c091dc072351151945a167e7968d2755559",
-                "sha256:2a9e864ec051a58fdb6bb2e6da03942adb20273897bc70067aee283e62bbac4d",
-                "sha256:2e0e2e01c5f61ddf47e3ed2d1fe1c9136e780ca6222d57a2517b9b02afd4710c",
-                "sha256:2e79eeeff8394284b09577f36316d410525e0cf0133abb3de10660e704d3d38e",
-                "sha256:2f2ac8bb01f705c5caaa7fe77ffd9b03f92f1b5061b94228f6ea5eaa0fca68ad",
-                "sha256:32819b662e3b4c26355a4403ea2f60c0a00db45b640fe722dd12db3d2ef807fb",
-                "sha256:3507c459767cf24c11e9520e2a37c89674266abe8e65453e5cb66398aa47ee7b",
-                "sha256:362faeae52dc6ccc50c0b6a01fa2ec0830bb61c292033f3749a46040b876f4ba",
-                "sha256:3650eae998dc718960e90120eb45d42bd57b18b21b10cb9ee05f91bff2345d48",
-                "sha256:36ff30385fb9fb3ac23a28bffdd4a230a5229ed5b15704b708b7c84bfb7fce51",
-                "sha256:3bc561c183684636c0099f9c3fbab8c1671841942edbce784bb01b4707d17924",
-                "sha256:3bd38b80491ef9686f719c1ad3d24d14fbd0e069988fdd4e7d1a6ffcdd7f4a13",
-                "sha256:3e37f1f134037601eb4b1f46854194f0cc082435dac2ee3de11e51529f7831f2",
-                "sha256:40f6e53461b19ddbb3354fe5bcf3d50d4333604ae4bf25b478333d83ca68002c",
-                "sha256:49db6c0a0e6626c2b97f5e7f8f7074da21cbd8ec73340c25e839a2457c007efa",
-                "sha256:4bcb1abecd998a72ad4e36a0fca93577fd0c059a6aacc44f16247031b98f6ff4",
-                "sha256:4cb55454a20d1b935f9eaab52e6ceab624a2efd8b52927c7ae7a43e02828dbe0",
-                "sha256:4f92d2372ec992c82fd7c74aa21e2a1910b3dcdc6a7e6392919a138f21d528a3",
-                "sha256:576d48e1e45c211e99fc02655ade65c32a75d3e383ccfd98ce59cece133ed02c",
-                "sha256:58bae860d1d116e6b4e1aad0cdc48a187d5893994f56d26db0c5534df7a47afd",
-                "sha256:5bb3f3cb6072c73e6ec1f865d8b80419b599f1597acf33f63fbf02252aab5a03",
-                "sha256:5db93f9017b384a4f194e1d89e1ce82d0a41b1fafdbbd3e0c8912baf13f2950f",
-                "sha256:5e41d5b334e8de4bc3f38843f31b2afa9a0c472ebf73119d3fd55cde08974bdf",
-                "sha256:60018626e637528a1fa64bb3a2b3e46ab7bf672052316d61c3629814d5e65052",
-                "sha256:6090ba604ea06b525a231450ae5d343917a393cbf50423900dea968daf61d16f",
-                "sha256:628fbb8be71a103499d10b189af7764996ab2634ed7b44b423f1e19901606e0e",
-                "sha256:6baea8a4f6f01e69e75cfdef3edd4a4d1c4b56238febbdf123ce96d09fbff010",
-                "sha256:6c5ca3eb817fb54bfd066740b64a2b31536eb8fe0b183dc35b09a7bd628ed680",
-                "sha256:70563a1596d2e0660ca2cebb738443437fc0e38597e7cbb276de0a7363924a52",
-                "sha256:7089d8bfa8064b28b2e39f5af7bf12d42f61caed884e35b9b4ea9e6fb1175077",
-                "sha256:72e9b1e92830c876cd49565d8404e4dcc9928302d348ea2517bc3f9e3a873a2a",
-                "sha256:7c7ca791bedda059e5195cf7c6b77384657a51429357cdd23e64ac1d4973d6dc",
-                "sha256:7f050ceffd8c730c1619a16bbf0b9cd037dcdb94b54710928ba38c7bde67e4a4",
-                "sha256:83da147124499fe41ed86edf34b4e81e951b3fe28edcc46288aac24e8a5c8484",
-                "sha256:86e8d6ff15fa7a9590c0addaf3ce52fb58bda4299cab2c2d0afa404db6848dab",
-                "sha256:8709eb4ab477c533b7d0a76cd3065d7d95c9e25e6b9f6e27caeeb8c63e8799c9",
-                "sha256:8e69bbe0ede8f7fe2616e779421bbdb37f025c802335a90f6416e4d98b368a37",
-                "sha256:8f90fc6dd505867514c8b8ef68a712dc0be90031a773c1ae2ad469f04062daef",
-                "sha256:9123ba0f3f98ff79780eebca9984a2b525f88563844b740f94cffb9099701230",
-                "sha256:927e3461dae0c09b1f2e0066e50c1a9204f8a64a3060f596e9a6742d3b307785",
-                "sha256:94876c21512535955a960f42a155213315e6ab06a4ce8ce372341a2a1b143eeb",
-                "sha256:98c0aecf661c175ce9cb17347fc51a5c98c3e9189ca57e8fcd9348dae18541db",
-                "sha256:9c7e7bd1fa1f535af71dfcd3700fc83a6dc261a1204f8f5327d8ffe82e52905d",
-                "sha256:9e7b3ad9f53ea9e085b3d27286dd13f8290969c0a153f8a52c8b5c46002c374b",
-                "sha256:9f9184744fb800c9f28e155a5896ecb54816296ee79d5d1978be6a2ae60f53c4",
-                "sha256:a3628815fd170a64624001bfb4e28946fd515bd672e68a1902d9e0290186eaf3",
-                "sha256:a5c330cb125983c5d380fef4a4155248a276297c86d64625fdaf500157e1981c",
-                "sha256:aa45cc71bf23a3181b8aa62466b5a2b7b7fb90fdc01df67ca433cd4fce7ec94d",
-                "sha256:aab24b9bbaa3d49e666e9309556591aa00748bd24ea74257a405f7fed9e8b10d",
-                "sha256:ac83f5228459b84fa6279e4126a53abfdd73cd9cc183947ee5084153880f65d7",
-                "sha256:ad21c60fc880204798f320387164dcacc25818a7b4ec2a0bf6b6c1d57b007d23",
-                "sha256:ae8a32ab77a84cc870bbfb60645851ca0f7d58fd251085ad67464b1445d632ca",
-                "sha256:b0f1d336786cb62613c72c00578c98e5bb8cd57b49c5bae5d4ab906ca7872f98",
-                "sha256:b28b9668a22ca2cfca4433441ba9acb2899624a323787a509a3dc5fbfa79c49d",
-                "sha256:b953d11b544ca5f2705bb77b177d8e17ab1bfd69e0fd99790a11549d2302258c",
-                "sha256:b9d8884d58ea8801e5906a491ab34af975091af76d1a389173db491ee7e316bb",
-                "sha256:ba3246c60303eab3d0e562addf25a983d60bddc36f4d1edc2510f056d19df255",
-                "sha256:bd0ad98c7d72b0e4cbfe89cdfa12cd07d2fd6ed22864341cdce12b318a383442",
-                "sha256:bf032367f921201deaecf221d4cc895ea84b3decf50a9c73ee106f961885a0ad",
-                "sha256:c31ecfc53ac03dad4928a1712f3a2893008bfba1b3cde49e1c14ff67faae2290",
-                "sha256:cbec8e43cace64e63398155dc585dc479a89fef1e57ead06c22d3441e1bd09c3",
-                "sha256:cc688a59c100f038fa9fec9e4ab457c2e2d1fca350fe7ea395016666f0d0a2dc",
-                "sha256:cd7da2adc721ccf19ac7ec86cae3a4fcaba03d9c477d5bd64ded6e9bb817bf3f",
-                "sha256:cd7e62e7d5bcfa38a62d8397fba6d0428b970ab7954c2197501cd1624f7f0bbb",
-                "sha256:d0f7f77a77c37159c9f417b8dd847f67a29e98c6acb52ee98fc6b91efbd1b2b6",
-                "sha256:d230fddc60caced271cc038e43e6fb8f4dd6b2dbaa44ac9763f2d76d05b0365a",
-                "sha256:d37f27ad80f742ef82796af3fe091888864958ad0bc8bab03da1830fa00c6004",
-                "sha256:d5ad7b1a1f6964d19b1a8acfc14bf7864f39587b3e25c16ca04f6cd1815026b3",
-                "sha256:d81359911c3bb31c899c6a5c23b403bdc0279215e5b3bc0d2a692489fed38632",
-                "sha256:d98802b78093c7083cc51f83da41a5be5a57d406798c9f69424bd75f8ae0812a",
-                "sha256:db0589e0bf41ff6ce284ab045ca89f27be1adf19e7bce26c2e7de6739a70c18b",
-                "sha256:ddbd113a37307638f94be5ae232a325155fd24dbfae2c56455da8724b471e7be",
-                "sha256:e3ece9aa6d07e18c966f14b4352a4c6f40249f6174d3d2c694c1062e19c6adbb",
-                "sha256:e3f9c9e5dd8eba4768e15f19044e1b5e216929a43a54b4ab329e103aed9f3eda",
-                "sha256:e41824343c2c129599645373992b1ce17720bb8a514f04ff9567031e1c26951e",
-                "sha256:e5dba1c11e089b526379e74f6c636202e4c5bad9a48c7416502b8a5b0d026c91",
-                "sha256:e791e3d13b14d0a7921804d0efe4d7bd15508bbcf8cb7a0c1ee1a27319a5f033",
-                "sha256:ec001689402b9104700b50a005c2d3d0218eae90eaa8bdbbd776fe78fe8a74b7",
-                "sha256:efffa359cc69840c8793f0c05a7b663de6afa7b9078fa6c80309ee38b9db677d",
-                "sha256:f1f191befea279cb9669b57be97ab1785781c8bab805900e95742ebfaa9cbf1d",
-                "sha256:f3331a3684192659fa1090bf2b448db928152fcba08222e58106f44758ef25f7",
-                "sha256:f40413d2859737ce6d95c29ce2dde0ef7cdc3063b5830ae4342fef5922c3bba7",
-                "sha256:f7ea49ddf51d5ec0c3cbd95190dd15e077a3153c8d4b22a33da43b5dd2b3c640",
-                "sha256:f82abb5c5b83dc30e96be99ce76239a030b62a73a13c64410e429660a5602bfd",
-                "sha256:fc20dadb102140dff63529e08ce6f9745dbd36e673ebb2b1c4a63e134bca81c2",
-                "sha256:fd37ab9a24021821b715478357af1cf369d5a42ac7405e83e5822be00732f463",
-                "sha256:ffd539d213c1ea2989ab92a5b9371ae7159c8c03cf2bcb9f2f594752f755ecd3"
+                "sha256:023574366002bf1bd751ebaf3e580aef4a468b3d3c216d2f3f7e16fdabd885ed",
+                "sha256:031f76fc87644a234883b51145e43985aa2d0c19b063e91d44379cd2786144f8",
+                "sha256:052a832078943d2b2627aea0d19381f607fe331cc0eb5df01991268253af8417",
+                "sha256:0699ab6b8c98df998c3eacf51a3b25864ca93dab157abe358af46dc95ecd9801",
+                "sha256:0713631d6e2d6c316c2f7b9320a34f44abb644fc487b77161d1724d883662e31",
+                "sha256:0774a46b38e70fdde0c6ded8d6d73115a7c39d7839a164cc833f170bbf539116",
+                "sha256:0898173249141ee99ffcd45e3829abe7bcee47d941af7434ccbf97717df020e5",
+                "sha256:09586f51a215d17efdb3a5f090d7cbf1633b7f3708f60a044757a5d48a83b393",
+                "sha256:102eac53bb0bf0f9a275b438e6cf6904904908562a1463a6fc3323cf47d7a532",
+                "sha256:10f32b53f424fc75ff7b713b2edb286fdbfc94bf16317890260a81c2c00385dc",
+                "sha256:150eec465dbc9cbca943c8e557a21afdcf9bab8aaabf386c44b794c2f94143d2",
+                "sha256:1d7360573f1e046cb3b0dceeb8864025aa78d98be4bb69f067ec1c40a9e2d9df",
+                "sha256:1f36a9d751f86455dc5278517e8b65580eeee37d61606183897f122c9e51cef3",
+                "sha256:24656dc36f866c33856baa3ab309da0b6a60f37d25d14be916bd3e79d9f3afcf",
+                "sha256:25860ed5c4e7f5e10c496ea78af46ae8d8468e0be745bd233bab9ca99bfd2647",
+                "sha256:26857f0f44f0e791f4a266595a7a09d21f6b589580ee0585f330aaccccb836e3",
+                "sha256:2bb2e4826be25e72013916eecd3d30f66fd076110de09f0e750163b416500721",
+                "sha256:2f6da6d842195fddc1cd34c3da8a40f6e99e4a113918faa5e60bf132f917c247",
+                "sha256:30adb75ecd7c2a52f5e76af50644b3e0b5ba036321c390b8e7ec1bb2a16dd43c",
+                "sha256:3339eca941568ed52d9ad0f1b8eb9fe0958fa245381747cecf2e9a78a5539c42",
+                "sha256:34ad87a831940521d462ac11f1774edf867c34172010f5390b2f06b85dcc6014",
+                "sha256:3777cc9dea0e6c464e4b24760664bd8831738cc582c1d8aacf1c3f546bef3f65",
+                "sha256:3953c6926a63f8ea5514644b7afb42659b505ece4183fdaaa8f61d978754349e",
+                "sha256:3c4eff26eddac49d52697a98ea01b0246e44ca82ab09354e94aae8823e8bda02",
+                "sha256:40578a6469e5d1df71b006936ce95804edb5df47b520c69cf5af264d462f2cbb",
+                "sha256:40f93086eef235623aa14dbddef1b9fb4b22b99454cb39a8d2e04c994fb9868c",
+                "sha256:4134aa2342f9b2ab6c33d5c172e40f9ef802c61bb9ca30d21782f6e035ed0043",
+                "sha256:442626328600bde1d09dc3bb00434f5374948838ce75c41a52152615689f9403",
+                "sha256:4a5ee600477b918ab345209eddafde9f91c0acd931f3776369585a1c55b04c57",
+                "sha256:4ce5a708d65a8dbf3748d2474b580d606b1b9f91b5c6ab2a316e0b0cf7a4ba50",
+                "sha256:516a611a2de12fbea70c78271e558f725c660ce38e0006f75139ba337d56b1f6",
+                "sha256:52c215eb46307c25f9fd2771cac8135d14b11a92ae48d17968eda5aa9aaf5071",
+                "sha256:53c43e10d398e365da2d4cc0bcaf0854b79b4c50ee9689652cdc72948e86f487",
+                "sha256:5752b761902cd15073a527b51de76bbae63d938dc7c5c4ad1e7d8df10e765138",
+                "sha256:5e8a78bd4879bff82daef48c14d5d4057f6856149094848c3ed0ecaf49f5aec2",
+                "sha256:5ed505ec6305abd2c2c9586a7b04fbd4baf42d4d684a9c12ec6110deefe2a063",
+                "sha256:5ee97c683eaface61d38ec9a489e353d36444cdebb128a27fe486a291647aff6",
+                "sha256:61fa268da6e2e1cd350739bb61011121fa550aa2545762e3dc02ea177ee4de35",
+                "sha256:64ccc28683666672d7c166ed465c09cee36e306c156e787acef3c0c62f90da5a",
+                "sha256:66414dafe4326bca200e165c2e789976cab2587ec71beb80f59f4796b786a238",
+                "sha256:68fe9199184c18d997d2e4293b34327c0009a78599ce703e15cd9a0f47349bba",
+                "sha256:6a555ae3d2e61118a9d3e549737bb4a56ff0cec88a22bd1dfcad5b4e04759175",
+                "sha256:6bdc11f9623870d75692cc33c59804b5a18d7b8a4b79ef0b00b773a27397d1f6",
+                "sha256:6cf4393c7b41abbf07c88eb83e8af5013606b1cdb7f6bc96b1b3536b53a574b8",
+                "sha256:6eef672de005736a6efd565577101277db6057f65640a813de6c2707dc69f396",
+                "sha256:734c41f9f57cc28658d98270d3436dba65bed0cfc730d115b290e970150c540d",
+                "sha256:73e0a78a9b843b8c2128028864901f55190401ba38aae685350cf69b98d9f7c9",
+                "sha256:775049dfa63fb58293990fc59473e659fcafd953bba1d00fc5f0631a8fd61977",
+                "sha256:7854a207ef77319ec457c1eb79c361b48807d252d94348305db4f4b62f40f7f3",
+                "sha256:78ca33811e1d95cac8c2e49cb86c0fb71f4d8409d8cbea0cb495b6dbddb30a55",
+                "sha256:79edd779cfc46b2e15b0830eecd8b4b93f1a96649bcb502453df471a54ce7977",
+                "sha256:7bf347b495b197992efc81a7408e9a83b931b2f056728529956a4d0858608b80",
+                "sha256:7fde6d0e00b2fd0dbbb40c0eeec463ef147819f23725eda58105ba9ca48744f4",
+                "sha256:81de24a1c51cfb32e1fbf018ab0bdbc79c04c035986526f76c33e3f9e0f3356c",
+                "sha256:879fb24304ead6b62dbe5034e7b644b71def53c70e19363f3c3be2705c17a3b4",
+                "sha256:8e7f2219cb72474571974d29a191714d822e58be1eb171f229732bc6fdedf0ac",
+                "sha256:9164ec8010327ab9af931d7ccd12ab8d8b5dc2f4c6a16cbdd9d087861eaaefa1",
+                "sha256:945eb4b6bb8144909b203a88a35e0a03d22b57aefb06c9b26c6e16d72e5eb0f0",
+                "sha256:99a57006b4ec39dbfb3ed67e5b27192792ffb0553206a107e4aadb39c5004cd5",
+                "sha256:9e9184fa6c52a74a5521e3e87badbf9692549c0fcced47443585876fcc47e469",
+                "sha256:9ff93d3aedef11f9c4540cf347f8bb135dd9323a2fc705633d83210d464c579d",
+                "sha256:a360cfd0881d36c6dc271992ce1eda65dba5e9368575663de993eeb4523d895f",
+                "sha256:a5d7ed104d158c0042a6a73799cf0eb576dfd5fc1ace9c47996e52320c37cb7c",
+                "sha256:ac17044876e64a8ea20ab132080ddc73b895b4abe9976e263b0e30ee5be7b9c2",
+                "sha256:ad857f42831e5b8d41a32437f88d86ead6c191455a3499c4b6d15e007936d4cf",
+                "sha256:b2039f8d545f20c4e52713eea51a275e62153ee96c8035a32b2abb772b6fc9e5",
+                "sha256:b455492cab07107bfe8711e20cd920cc96003e0da3c1f91297235b1603d2aca7",
+                "sha256:b4a9fe992887ac68256c930a2011255bae0bf5ec837475bc6f7edd7c8dfa254e",
+                "sha256:b5a53f5998b4bbff1cb2e967e66ab2addc67326a274567697379dd1e326bded7",
+                "sha256:b788276a3c114e9f51e257f2a6f544c32c02dab4aa7a5816b96444e3f9ffc336",
+                "sha256:bddd4f91eede9ca5275e70479ed3656e76c8cdaaa1b354e544cbcf94c6fc8ac4",
+                "sha256:c0503c5b681566e8b722fe8c4c47cce5c7a51f6935d5c7012c4aefe952a35eed",
+                "sha256:c1b3cd23d905589cb205710b3988fc8f46d4a198cf12862887b09d7aaa6bf9b9",
+                "sha256:c48f3fbc3e92c7dd6681a258d22f23adc2eb183c8cb1557d2fcc5a024e80b094",
+                "sha256:c63c3ef43f0b3fb00571cff6c3967cc261c0ebd14a0a134a12e83bdb8f49f21f",
+                "sha256:c6c45a2d2b68c51fe3d9352733fe048291e483376c94f7723458cfd7b473136b",
+                "sha256:caa1afc70a02645809c744eefb7d6ee8fef7e2fad170ffdeacca267fd2674f13",
+                "sha256:cc435d059f926fdc5b05822b1be4ff2a3a040f3ae0a7bbbe672babb468944722",
+                "sha256:cf693eb4a08eccc1a1b636e4392322582db2a47470d52e824b25eca7a3977b53",
+                "sha256:cf71343646756a072b85f228d35b1d7407da1669a3de3cf47f8bbafe0c8183a4",
+                "sha256:d08f63561c8a695afec4975fae445245386d645e3e446e6f260e81663bfd2e38",
+                "sha256:d29ddefeab1791e3c751e0189d5f4b3dbc0bbe033b06e9c333dca1f99e1d523e",
+                "sha256:d7f5e15c953ace2e8dde9824bdab4bec50adb91a5663df08d7d994240ae6fa31",
+                "sha256:d858532212f0650be12b6042ff4378dc2efbb7792a286bee4489eaa7ba010586",
+                "sha256:d97dd44683802000277bbf142fd9f6b271746b4846d0acaf0cefa6b2eaf2a7ad",
+                "sha256:dcdc88b6b01015da066da3fb76545e8bb9a6880a5ebf89e0f0b2e3ca557b3ab7",
+                "sha256:dd609fafdcdde6e67a139898196698af37438b035b25ad63704fd9097d9a3482",
+                "sha256:defa2c0c68734f4a82028c26bcc85e6b92cced99866af118cd6a89b734ad8e0d",
+                "sha256:e22260a4741a0e7a206e175232867b48a16e0401ef5bce3c67ca5b9705879066",
+                "sha256:e225a6a14ecf44499aadea165299092ab0cba918bb9ccd9304eab1138844490b",
+                "sha256:e3df0bc35e746cce42579826b89579d13fd27c3d5319a6afca9893a9b784ff1b",
+                "sha256:e6fcc026a3f27c1282c7ed24b7fcac82cdd70a0e84cc848c0841a3ab1e3dea2d",
+                "sha256:e782379c2028a3611285a795b89b99a52722946d19fc06f002f8b53e3ea26ea9",
+                "sha256:e8cdd52744f680346ff8c1ecdad5f4d11117e1724d4f4e1874f3a67598821069",
+                "sha256:e9616f5bd2595f7f4a04b67039d890348ab826e943a9bfdbe4938d0eba606971",
+                "sha256:e98c4c07ee4c4b3acf787e91b27688409d918212dfd34c872201273fdd5a0e18",
+                "sha256:ebdab79f42c5961682654b851f3f0fc68e6cc7cd8727c2ac4ffff955154123c1",
+                "sha256:f0f17f2ce0f3529177a5fff5525204fad7b43dd437d017dd0317f2746773443d",
+                "sha256:f4e56860a5af16a0fcfa070a0a20c42fbb2012eed1eb5ceeddcc7f8079214281"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.10.4"
+            "version": "==0.10.6"
         },
         "s3transfer": {
             "hashes": [
@@ -2299,7 +2394,7 @@
                 "sha256:febffa5b1eda6622d44b245b0685aff6fb555ce0ed734e2d7b1c3acd018a2cff",
                 "sha256:ff836cd4041e16003549449cc0a5e372f6b6f871eb89007ab0ee18fb2800fded"
             ],
-            "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==3.19.2"
         },
         "six": {
@@ -2307,7 +2402,7 @@
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==1.16.0"
         },
         "smmap": {
@@ -2346,16 +2441,23 @@
                 "sha256:2e126cf98b7fd38f1e33c64484406b78e937b1a280e078ef558b95bf5b6895f6",
                 "sha256:36e58f8c4fe43984384e3fbe6341ac99b6b4e083de2fe838f0fdb91cebe9e9cb",
                 "sha256:37ce517c011560d68f1ffb28af65d7e06f873f191eb3a73af5671e9c3fada08a",
+                "sha256:393cd06c3b00b57f5421e2133e088df9cabcececcea180327e43b937b5a7caa5",
                 "sha256:45806315aae81a0c202752558f0df52b42d11dd7ba0097bf71e253b4215f34f4",
                 "sha256:4afbbf5ef41ac18e02c8dc1f86c04b22b7a2125f2a030e25bbb4aff31abb224b",
                 "sha256:5debe7d49b8acf1f3035317e63d9ec8d5e4d904c6e75a2a9246a119f5f2fdf3d",
                 "sha256:5fb1ebdfc8373b5a291485757bd6431de8d7ed42c27439f543c81f6c8febd729",
                 "sha256:647e0b309cb4512b1f1b78471fdaf72921b6fa6e750b9f891e09c6e2f0e5326f",
+                "sha256:66da9627cfcc43bbdebd47bfe0145bb662041472393c03b7802253993b6b7c90",
                 "sha256:706bfa02157b97c136547c406f263e4c6274a7b061b3eb9742915dd774bbc264",
+                "sha256:738d7321212941ab19ba2acf02a68b8ee64987b248ffa2101630e8fccb549e0d",
                 "sha256:7653ed6817c710d0c95558232aba799307d14ae084cc9b1f4c389157ec50df5c",
+                "sha256:7cf8b90ad84ad3a45098b1c9f56f2b161601e4670827d6b892ea0e884569bd1d",
                 "sha256:82b08e82da3756765c2e75f327b9bf6b0f043c9c3925fb95fb51e1567fa4ee87",
+                "sha256:8396e896e08e37032e87e7fbf4a15f431aa878c286dc7f79e616c2feacdb366c",
                 "sha256:8923dfdf24d5aa8a3adb59723f54118dd4fe62cf59ed0d0d65d940579c1170a4",
+                "sha256:95ab792ca493891d7a45a077e35b418f68435efb3e1706cb8155e20e86a9013c",
                 "sha256:95b9df9afd680b7a3b13b38adf6e3a38995da5e162cc7524ef08e3be4e5ed3e1",
+                "sha256:9a06e046ffeb8a484279e54bda0a5abfd9675f594a2e38ef3133d7e4d75b6214",
                 "sha256:9c21b172dfb22e0db303ff6419451f0cac891d2e911bb9fbf8003d717f1bcf91",
                 "sha256:a1878ce508edea4a879015ab5215546c444233881301e97ca16fe251e89f1c55",
                 "sha256:a63e43bf3f668c11bb0444ce6e809c1227b8f067ca1068898f3008a273f52b09",
@@ -2368,10 +2470,13 @@
                 "sha256:bbdf16372859b8ed3f4d05f925a984771cd2abd18bd187042f24be4886c2a15f",
                 "sha256:c14b29d9e1529f99efd550cd04dbb6db6ba5d690abb96d52de2bff4ed518bc95",
                 "sha256:c40f3470e084d31247aea228aa1c39bbc0904c2b9ccbf5d3cfa2ea2dac06f26d",
+                "sha256:ca46de16650d143a928d10842939dab208e8d8c3a9a8757600cae9b7c579c5cd",
                 "sha256:ccf956da45290df6e809ea12c54c02ace7f8ff4d765d6d3dfb3655ee876ce58d",
                 "sha256:d26f280b8f0a8f497bc10573849ad6dc62e671d2468826e5c748d04ed9e670d5",
+                "sha256:ebc22807a7e161c0d8f3da34018ab7c97ef6223578fcdd99b1d3e7ed1100a5db",
                 "sha256:ec2268de67f73b43320383947e74700e95c6770d0c68c4e615e9897e46296294",
                 "sha256:f167c8175ab908ce48bd6550679cc6ea20ae169379e73c7720a28f89e53aa532",
+                "sha256:f23755c384c2969ca2f7667a83f7c5648fcf8b62a3f2bbd883d805454964a800",
                 "sha256:f835c050ebaa4e48b18403bed2c0fda986525896efd76c245bdd4db995e51a4c",
                 "sha256:f8a65990c9c490f4651b5c02abccc9f113a7f56fa482031ac8cb88b70bc8ccaa"
             ],
@@ -2704,11 +2809,11 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21",
-                "sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b"
+                "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84",
+                "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
-            "version": "==1.26.17"
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0.7"
         },
         "uvicorn": {
             "hashes": [
@@ -2741,11 +2846,11 @@
         },
         "websocket-client": {
             "hashes": [
-                "sha256:3aad25d31284266bcfcfd1fd8a743f63282305a364b8d0948a43bd606acc652f",
-                "sha256:6cfc30d051ebabb73a5fa246efdcc14c8fbebbd0330f8984ac3bb6d9edd2ad03"
+                "sha256:084072e0a7f5f347ef2ac3d8698a5e0b4ffbfcab607628cadabc650fc9a83a24",
+                "sha256:b3324019b3c28572086c4a319f91d1dcd44e6e11cd340232978c684a7650d0df"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.6.3"
+            "version": "==1.6.4"
         },
         "xxhash": {
             "hashes": [
@@ -3265,11 +3370,11 @@
         },
         "pycodestyle": {
             "hashes": [
-                "sha256:259bcc17857d8a8b3b4a2327324b79e5f020a13c16074670f9c8c8f872ea76d0",
-                "sha256:5d1013ba8dc7895b548be5afb05740ca82454fd899971563d2ef625d090326f8"
+                "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f",
+                "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.11.0"
+            "version": "==2.11.1"
         },
         "pyflakes": {
             "hashes": [
@@ -3336,11 +3441,11 @@
         },
         "types-requests": {
             "hashes": [
-                "sha256:39894cbca3fb3d032ed8bdd02275b4273471aa5668564617cc1734b0a65ffdf8",
-                "sha256:e1b325c687b3494a2f528ab06e411d7092cc546cc9245c000bacc2fca5ae96d4"
+                "sha256:b32b9a86beffa876c0c3ac99a4cd3b8b51e973fb8e3bd4e0a6bb32c7efad80fc",
+                "sha256:dc5852a76f1eaf60eafa81a2e50aefa3d1f015c34cf0cba130930866b1b22a92"
             ],
             "index": "pypi",
-            "version": "==2.31.0.8"
+            "version": "==2.31.0.10"
         },
         "types-toml": {
             "hashes": [
@@ -3360,11 +3465,11 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21",
-                "sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b"
+                "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84",
+                "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
-            "version": "==1.26.17"
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0.7"
         },
         "wheel": {
             "hashes": [
diff --git a/documentation/docs/tooltips/experiments/_deepspeed-offload-optimizer.mdx b/documentation/docs/tooltips/experiments/_deepspeed-offload-optimizer.mdx
new file mode 100644
index 000000000..d83fa269a
--- /dev/null
+++ b/documentation/docs/tooltips/experiments/_deepspeed-offload-optimizer.mdx
@@ -0,0 +1 @@
+Whether to offload optimizer to cpu for saving more GPU ram during training. Note that turn on offload_optimizer would further slow down training.
\ No newline at end of file
diff --git a/documentation/docs/tooltips/experiments/_deepspeed-reduce-bucket-size.mdx b/documentation/docs/tooltips/experiments/_deepspeed-reduce-bucket-size.mdx
new file mode 100644
index 000000000..d2b6d37eb
--- /dev/null
+++ b/documentation/docs/tooltips/experiments/_deepspeed-reduce-bucket-size.mdx
@@ -0,0 +1 @@
+Number of elements reduced/allreduced at a time. Limits the memory required for the allgather for large model sizes. Smaller values use less memory, but slow down training.
\ No newline at end of file
diff --git a/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-live-parameters.mdx b/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-live-parameters.mdx
new file mode 100644
index 000000000..708141121
--- /dev/null
+++ b/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-live-parameters.mdx
@@ -0,0 +1 @@
+The maximum number of parameters resident per GPU before releasing. Smaller values use less memory, but slow down training.
\ No newline at end of file
diff --git a/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-reuse-distance.mdx b/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-reuse-distance.mdx
new file mode 100644
index 000000000..0e5beb51a
--- /dev/null
+++ b/documentation/docs/tooltips/experiments/_deepspeed-stage3-max-reuse-distance.mdx
@@ -0,0 +1 @@
+Do not release a parameter if it will be reused within this threshold of parameters. Smaller values use less memory, but slow down training.
\ No newline at end of file
diff --git a/documentation/docs/tooltips/experiments/_deepspeed-stage3-param-persistence-threshold.mdx b/documentation/docs/tooltips/experiments/_deepspeed-stage3-param-persistence-threshold.mdx
new file mode 100644
index 000000000..ab7d46121
--- /dev/null
+++ b/documentation/docs/tooltips/experiments/_deepspeed-stage3-param-persistence-threshold.mdx
@@ -0,0 +1 @@
+Do not partition parameters smaller than this threshold. Smaller values use less memory, but can greatly increase communication and slow down training. (especially latency-bound messages).
diff --git a/documentation/docs/tooltips/experiments/_deepspeed-stage3-prefetch-bucket-size.mdx b/documentation/docs/tooltips/experiments/_deepspeed-stage3-prefetch-bucket-size.mdx
new file mode 100644
index 000000000..eb24ae37d
--- /dev/null
+++ b/documentation/docs/tooltips/experiments/_deepspeed-stage3-prefetch-bucket-size.mdx
@@ -0,0 +1 @@
+Maximum number of parameter elements to fetch ahead of use. Smaller values use less memory, but slow down training..
\ No newline at end of file
diff --git a/documentation/docs/tooltips/experiments/_use-deepspeed.mdx b/documentation/docs/tooltips/experiments/_use-deepspeed.mdx
new file mode 100644
index 000000000..6865a6fd0
--- /dev/null
+++ b/documentation/docs/tooltips/experiments/_use-deepspeed.mdx
@@ -0,0 +1 @@
+Whether to use deepspeed for saving GPU ram during training. Note that turning on DeepSpeed can slow down training.
\ No newline at end of file
diff --git a/llm_studio/app_utils/sections/experiment.py b/llm_studio/app_utils/sections/experiment.py
index 7d8fdcf66..4b8e2a1b9 100644
--- a/llm_studio/app_utils/sections/experiment.py
+++ b/llm_studio/app_utils/sections/experiment.py
@@ -1680,7 +1680,9 @@ async def experiment_push_to_huggingface_dialog(q: Q, error: str = ""):
         num_running_queued = len(
             experiments[experiments["status"].isin(["queued", "running"])]
         )
-        if num_running_queued > 0:
+        experiment_path = q.client["experiment/display/experiment_path"]
+        cfg = load_config_yaml(os.path.join(experiment_path, "cfg.yaml"))
+        if num_running_queued > 0 or cfg.environment.use_deepspeed:
             default_device = "cpu"
 
         try:
diff --git a/llm_studio/app_utils/utils.py b/llm_studio/app_utils/utils.py
index f10a7ece3..a7e83f415 100644
--- a/llm_studio/app_utils/utils.py
+++ b/llm_studio/app_utils/utils.py
@@ -114,36 +114,41 @@ def start_process(
     env = {**os.environ, **env_vars}
 
     if num_gpus == 0:
-        p = subprocess.Popen(
-            [
-                "python",
-                "train_wave.py",
-                "-Y",
-                config_name,
-                "-Q",
-                ",".join([str(x) for x in process_queue]),
-            ],
-            env=env,
-        )
+        cmd = [
+            "python",
+            "train_wave.py",
+            "-Y",
+            config_name,
+        ]
     # Do not delete for debug purposes
     # elif num_gpus == 1:
-    #     p = subprocess.Popen(
-    #         [
-    #             "env",
-    #             f"CUDA_VISIBLE_DEVICES={','.join(gpu_list)}",
-    #             "python",
-    #             "-u",
-    #             "train_wave.py",
-    #             "-P",
-    #             config_name,
-    #             "-Q",
-    #             ",".join([str(x) for x in process_queue]),
-    #         ]
-    #     )
+    #     cmd = [
+    #         "env",
+    #         f"CUDA_VISIBLE_DEVICES={','.join(gpu_list)}",
+    #         "python",
+    #         "-u",
+    #         "train_wave.py",
+    #         "-P",
+    #         config_name,
+    #     ]
     else:
         free_port = find_free_port()
-        p = subprocess.Popen(
-            [
+        if cfg.environment.use_deepspeed:
+            logger.info("Starting deepspeed...")
+            cmd = [
+                "env",
+                "deepspeed",
+                "--include",
+                f"localhost:{','.join(gpu_list)}",
+                "--master_port",
+                f"{str(free_port)}",
+                "train_wave.py",
+                "-Y",
+                config_name,
+            ]
+        else:
+            logger.info("Starting torchrun...")
+            cmd = [
                 "env",
                 f"CUDA_VISIBLE_DEVICES={','.join(gpu_list)}",
                 "torchrun",
@@ -152,11 +157,17 @@ def start_process(
                 "train_wave.py",
                 "-Y",
                 config_name,
-                "-Q",
-                ",".join([str(x) for x in process_queue]),
-            ],
-            env=env,
-        )
+            ]
+
+    if len(process_queue) > 0:
+        cmd.append("-Q")
+        cmd.append(",".join([str(x) for x in process_queue]))
+
+    p = subprocess.Popen(
+        cmd,
+        env=env,
+    )
+
     logger.info(f"Percentage of RAM memory used: {psutil.virtual_memory().percent}")
 
     return p
diff --git a/llm_studio/python_configs/text_causal_language_modeling_config.py b/llm_studio/python_configs/text_causal_language_modeling_config.py
index 77f9549e0..83b0aeed6 100644
--- a/llm_studio/python_configs/text_causal_language_modeling_config.py
+++ b/llm_studio/python_configs/text_causal_language_modeling_config.py
@@ -227,9 +227,9 @@ class ConfigNLPCausalLMTokenizer(DefaultConfig):
 
     def __post_init__(self):
         super().__post_init__()
-        self._possible_values["max_length_prompt"] = (32, 8192, 32)
-        self._possible_values["max_length_answer"] = (32, 8192, 32)
-        self._possible_values["max_length"] = (32, 8192, 32)
+        self._possible_values["max_length_prompt"] = (32, 1024 * 16, 32)
+        self._possible_values["max_length_answer"] = (32, 1024 * 16, 32)
+        self._possible_values["max_length"] = (32, 1024 * 16, 32)
         self._possible_values["padding_quantile"] = (0, 1, 0.01)
         self._padding_side = "left"
 
@@ -343,6 +343,13 @@ class ConfigNLPCausalLMEnvironment(DefaultConfig):
 
     compile_model: bool = False
     use_fsdp: bool = False
+    use_deepspeed: bool = False
+    deepspeed_reduce_bucket_size: int = 1e6
+    deepspeed_stage3_prefetch_bucket_size: int = 1e6
+    deepspeed_stage3_param_persistence_threshold: int = 1e6
+    #     deepspeed_offload_optimizer: bool = False
+    #     deepspeed_stage3_max_live_parameters: int = 1e9
+    #     deepspeed_stage3_max_reuse_distance: int = 1e9
 
     find_unused_parameters: bool = False
     trust_remote_code: bool = True
@@ -376,6 +383,37 @@ def __post_init__(self):
 
         self._possible_values["number_of_workers"] = (1, multiprocessing.cpu_count(), 1)
         self._possible_values["seed"] = possible_values.Number(step=1, min=-1)
+        self._possible_values["deepspeed_reduce_bucket_size"] = possible_values.Number(
+            step=1, min=1e6
+        )
+        self._possible_values[
+            "deepspeed_stage3_prefetch_bucket_size"
+        ] = possible_values.Number(step=1, min=1e6)
+        self._possible_values[
+            "deepspeed_stage3_param_persistence_threshold"
+        ] = possible_values.Number(step=1, min=1e6)
+        self._possible_values[
+            "deepspeed_stage3_max_live_parameters"
+        ] = possible_values.Number(step=1, min=1e6)
+        self._possible_values[
+            "deepspeed_stage3_max_reuse_distance"
+        ] = possible_values.Number(step=1, min=1e6)
+        self._nesting.add(
+            [
+                "deepspeed_reduce_bucket_size",
+                "deepspeed_stage3_prefetch_bucket_size",
+                "deepspeed_stage3_param_persistence_threshold",
+                # "deepspeed_offload_optimizer",
+            ],
+            [Dependency(key="use_deepspeed", value=False, is_set=False)],
+        )
+        # self._nesting.add(
+        #     [
+        #         "deepspeed_stage3_max_live_parameters",
+        #         "deepspeed_stage3_max_reuse_distance",
+        #     ],
+        #     [Dependency(key="deepspeed_offload_optimizer", value=False, is_set=False)],  # noqa: E501
+        # )
 
 
 @dataclass
diff --git a/llm_studio/src/utils/modeling_utils.py b/llm_studio/src/utils/modeling_utils.py
index 34e50a0e9..db0c36891 100644
--- a/llm_studio/src/utils/modeling_utils.py
+++ b/llm_studio/src/utils/modeling_utils.py
@@ -7,8 +7,11 @@
 from typing import Any, Dict
 
 import coolname
+import deepspeed
 import numpy as np
 import torch
+from deepspeed.runtime.dataloader import DeepSpeedDataLoader
+from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
 from peft import LoraConfig, get_peft_model
 from torch.cuda.amp import autocast
 from torch.distributed.fsdp.fully_sharded_data_parallel import (
@@ -31,6 +34,7 @@
 from llm_studio.src.optimizers import Optimizers
 from llm_studio.src.schedulers import Schedulers
 from llm_studio.src.utils.data_utils import (
+    OrderedDistributedSampler,
     batch_padding,
     cat_batches,
     get_inference_batch_size,
@@ -51,7 +55,7 @@ def unwrap_model(model: torch.nn.Module):
     return model
 
 
-def check_disk_space(model: torch.nn.Module, path: str):
+def check_disk_space(model: torch.nn.Module, path: str, use_deepspeed: bool = False):
     total, used, free = shutil.disk_usage(path)
 
     model_size_in_bytes = 0
@@ -67,6 +71,8 @@ def check_disk_space(model: torch.nn.Module, path: str):
             model_size_in_bytes += param.numel() * 4
             logger.warning(f"Unsupported data type: {param.data.dtype}")
 
+    if use_deepspeed:
+        model_size_in_bytes *= 2  # need double space for converting deepspeed engine.
     if model_size_in_bytes * 1.03 < free:  # leave a 3% margin here.
         logger.info("Enough space available for saving model weights.")
     else:
@@ -89,15 +95,25 @@ def save_checkpoint(model: torch.nn.Module, path: str, cfg: Any):
         Dictionary with all the keys to save
     """
 
-    model = unwrap_model(model)
-
-    if hasattr(cfg.training, "lora") and cfg.training.lora:
-        model.backbone.save_pretrained(path)
-
-    checkpoint = {"model": model.state_dict()}
-
-    if path is not None:
-        torch.save(checkpoint, os.path.join(path, "checkpoint.pth"))
+    if cfg.environment.use_deepspeed:
+        if path is not None:
+            # gather model params from all ranks
+            model.save_checkpoint(os.path.join(path, "ds_checkpoint"))
+            if cfg.environment._local_rank == 0:
+                # load to cpu
+                state_dict = get_fp32_state_dict_from_zero_checkpoint(
+                    os.path.join(path, "ds_checkpoint")
+                )
+                # save as normal checkpoint that can be loaded by `load_state_dict`
+                checkpoint = {"model": state_dict}
+                torch.save(checkpoint, os.path.join(path, "checkpoint.pth"))
+                shutil.rmtree(os.path.join(path, "ds_checkpoint"))
+    else:
+        if cfg.environment._local_rank == 0:
+            model = unwrap_model(model)
+            checkpoint = {"model": model.state_dict()}
+            if path is not None:
+                torch.save(checkpoint, os.path.join(path, "checkpoint.pth"))
 
     if cfg.problem_type == "text_causal_classification_modeling":
         torch.save(
@@ -189,8 +205,63 @@ def load_checkpoint(
         logger.info(f"Weights loaded from: {weights_path}")
 
 
-def wrap_model_distributed(model: torch.nn.Module, cfg: Any, fsdp: bool):
-    if fsdp:
+def get_ds_config(cfg: Any):
+    ds_config = {
+        "fp16": {
+            "enabled": True if cfg.architecture.backbone_dtype == "float16" else False,
+            "loss_scale_window": 100,
+        },
+        "bf16": {
+            "enabled": True if cfg.architecture.backbone_dtype == "bfloat16" else False,
+            "loss_scale_window": 100,
+        },
+        # https://www.deepspeed.ai/docs/config-json/#zero-optimizations-for-fp16-training
+        "zero_force_ds_cpu_optimizer": False,
+        "zero_optimization": {
+            "stage": 3,
+            "overlap_comm": True,
+            "contiguous_gradients": True,
+            "reduce_bucket_size": cfg.environment.deepspeed_reduce_bucket_size,
+            # zero3
+            "stage3_prefetch_bucket_size": cfg.environment.deepspeed_stage3_prefetch_bucket_size,  # noqa: E501
+            "stage3_param_persistence_threshold": cfg.environment.deepspeed_stage3_param_persistence_threshold,  # noqa: E501
+            # zero3 offload cpu
+            # "stage3_max_live_parameters": cfg.environment.deepspeed_stage3_max_live_parameters,  # noqa: E501
+            # "stage3_max_reuse_distance": cfg.environment.deepspeed_stage3_max_reuse_distance,  # noqa: E501
+            # zero++
+            # "reduce_scatter": True,
+            # "zero_quantized_weights": True,
+            # "zero_hpz_partition_size": 16,
+            # "zero_quantized_gradients": True,
+        },
+        "steps_per_print": 2000,
+        "train_micro_batch_size_per_gpu": cfg.training.batch_size,
+        "gradient_accumulation_steps": cfg.training.grad_accumulation,
+        "wall_clock_breakdown": False,
+    }
+    # TODO: Do not enable offload cpu for now.
+    # if cfg.environment.deepspeed_offload_optimizer:
+    #     ds_config["zero_optimization"]["offload_optimizer"] = {
+    #         "device": "cpu",
+    #         "pin_memory": True,
+    #     }
+    # TODO: RuntimeError: Tensors must be CUDA and dense
+    # if cfg.environment.deepspeed_offload_param:
+    #     ds_config["zero_optimization"]["offload_param"] =
+    #         {"device": "cpu", "pin_memory": True}
+
+    return ds_config
+
+
+def wrap_model_distributed(
+    model: torch.nn.Module,
+    optimizer: torch.optim.Optimizer,
+    lr_scheduler: torch.optim.lr_scheduler._LRScheduler,
+    train_dataloader: torch.utils.data.DataLoader,
+    val_dataloader: torch.utils.data.DataLoader,
+    cfg: Any,
+):
+    if cfg.environment.use_fsdp:
         auto_wrap_policy = None
 
         mixed_precision_policy = None
@@ -211,6 +282,27 @@ def wrap_model_distributed(model: torch.nn.Module, cfg: Any, fsdp: bool):
             # use_orig_params=False
             limit_all_gathers=True,
         )
+    elif cfg.environment.use_deepspeed:
+        ds_config = get_ds_config(cfg)
+        model, optimizer, train_dataloader, lr_scheduler = deepspeed.initialize(
+            model=model,
+            optimizer=optimizer,
+            lr_scheduler=lr_scheduler,
+            training_data=train_dataloader.dataset,
+            config_params=ds_config,
+        )
+        val_dataloader = DeepSpeedDataLoader(
+            val_dataloader.dataset,
+            batch_size=val_dataloader.batch_size,
+            local_rank=cfg.environment._local_rank,
+            pin_memory=True,
+            tput_timer=None,
+            data_sampler=OrderedDistributedSampler(
+                val_dataloader.dataset,
+                num_replicas=cfg.environment._world_size,
+                rank=cfg.environment._local_rank,
+            ),
+        )
     else:
         find_unused_parameters = cfg.environment.find_unused_parameters
         if getattr(cfg.architecture, "gradient_checkpointing", None):
@@ -221,7 +313,7 @@ def wrap_model_distributed(model: torch.nn.Module, cfg: Any, fsdp: bool):
             find_unused_parameters=find_unused_parameters,
         )
 
-    return model
+    return model, optimizer, train_dataloader, val_dataloader, lr_scheduler
 
 
 def get_optimizer(model: torch.nn.Module, cfg: Any) -> torch.optim.Optimizer:
@@ -234,7 +326,6 @@ def get_optimizer(model: torch.nn.Module, cfg: Any) -> torch.optim.Optimizer:
     Returns:
         Optimizer
     """
-
     no_decay = ["bias", "LayerNorm.weight"]
     differential_layers = cfg.training.differential_learning_rate_layers
     optimizer = Optimizers.get(cfg.training.optimizer)(
@@ -245,7 +336,7 @@ def get_optimizer(model: torch.nn.Module, cfg: Any) -> torch.optim.Optimizer:
                     for name, param in model.named_parameters()
                     if (not any(layer in name for layer in differential_layers))
                     and (not any(nd in name for nd in no_decay))
-                    # and param.requires_grad
+                    and param.requires_grad
                 ],
                 "lr": cfg.training.learning_rate,
                 "weight_decay": cfg.training.weight_decay,
@@ -256,7 +347,7 @@ def get_optimizer(model: torch.nn.Module, cfg: Any) -> torch.optim.Optimizer:
                     for name, param in model.named_parameters()
                     if (not any(layer in name for layer in differential_layers))
                     and (any(nd in name for nd in no_decay))
-                    # and param.requires_grad
+                    and param.requires_grad
                 ],
                 "lr": cfg.training.learning_rate,
                 "weight_decay": 0,
@@ -267,7 +358,7 @@ def get_optimizer(model: torch.nn.Module, cfg: Any) -> torch.optim.Optimizer:
                     for name, param in model.named_parameters()
                     if (any(layer in name for layer in differential_layers))
                     and (not any(nd in name for nd in no_decay))
-                    # and param.requires_grad
+                    and param.requires_grad
                 ],
                 "lr": cfg.training.differential_learning_rate,
                 "weight_decay": cfg.training.weight_decay,
@@ -278,7 +369,7 @@ def get_optimizer(model: torch.nn.Module, cfg: Any) -> torch.optim.Optimizer:
                     for name, param in model.named_parameters()
                     if (any(layer in name for layer in differential_layers))
                     and (any(nd in name for nd in no_decay))
-                    # and param.requires_grad
+                    and param.requires_grad
                 ],
                 "lr": cfg.training.differential_learning_rate,
                 "weight_decay": 0,
@@ -423,15 +514,23 @@ def run_inference(
 
         batch = cfg.dataset.dataset_class.batch_to_device(data, cfg.environment._device)
 
-        with autocast(enabled=cfg.environment.mixed_precision):
-            output = model.forward(batch)
-            if (
-                cfg.prediction.metric != "Perplexity"
-                and cfg.problem_type != "text_causal_classification_modeling"
-            ):
+        if cfg.environment.use_deepspeed:
+            if cfg.prediction.metric != "Perplexity":
+                output = {}
                 output["predicted_answer_ids"] = (
-                    unwrap_model(model).generate(batch, cfg).detach().cpu()
+                    model.module.generate(batch, cfg).detach().cpu()
                 )
+            else:
+                output = model.forward(batch)
+        else:
+            with autocast(enabled=cfg.environment.mixed_precision):
+                if cfg.prediction.metric != "Perplexity":
+                    output = {}
+                    output["predicted_answer_ids"] = (
+                        unwrap_model(model).generate(batch, cfg).detach().cpu()
+                    )
+                else:
+                    output = model.forward(batch)
         if contains_nan(output) and cfg.environment.mixed_precision:
             raise LLMModelException(
                 "NaN caught during mixed precision inference. "
diff --git a/requirements.txt b/requirements.txt
index 2c86b38d7..61e7b44ed 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,35 +1,36 @@
 -i https://pypi.org/simple
 --extra-index-url https://download.pytorch.org/whl/cu118
 accelerate==0.23.0
-aiohttp==3.8.5 ; python_version >= '3.6'
+aiohttp==3.8.6 ; python_version >= '3.6'
 aiosignal==1.3.1 ; python_version >= '3.7'
 anyio==4.0.0 ; python_version >= '3.8'
 arrow==1.3.0 ; python_version >= '3.8'
 async-timeout==4.0.3 ; python_version >= '3.7'
 attrs==23.1.0 ; python_version >= '3.7'
 azure-core==1.29.4 ; python_version >= '3.7'
-azure-storage-blob==12.18.2 ; python_version >= '3.7'
-azure-storage-file-datalake==12.13.1
+azure-storage-blob==12.18.3 ; python_version >= '3.7'
+azure-storage-file-datalake==12.13.2
 beautifulsoup4==4.12.2
 bitsandbytes==0.41.1
-bleach==6.0.0 ; python_version >= '3.7'
+bleach==6.1.0 ; python_version >= '3.8'
 blessed==1.20.0 ; python_version >= '2.7'
-bokeh==3.2.2
-boto3==1.28.61
-botocore==1.31.61 ; python_version >= '3.7'
+bokeh==3.3.0
+boto3==1.28.65
+botocore==1.31.65 ; python_version >= '3.7'
 bravado==11.0.3 ; python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_full_version != '3.5.0'
 bravado-core==6.1.0 ; python_version >= '3.7'
 certifi==2023.7.22 ; python_version >= '3.6'
 cffi==1.16.0 ; python_version >= '3.8'
 charset-normalizer==3.3.0 ; python_full_version >= '3.7.0'
 click==8.1.7 ; python_version >= '3.7'
-cmake==3.27.6
+cmake==3.27.7
 colorama==0.4.6 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
 contourpy==1.1.1 ; python_version >= '3.8'
 coolname==2.2.0
 cramjam==2.7.0 ; python_version >= '3.7'
 cryptography==41.0.4 ; python_version >= '3.7'
 datasets==2.14.5
+deepspeed==0.11.1
 dill==0.3.7
 einops==0.6.1
 exceptiongroup==1.1.3 ; python_version < '3.11'
@@ -38,14 +39,15 @@ filelock==3.12.4 ; python_version >= '3.8'
 fqdn==1.5.1
 frozenlist==1.4.0 ; python_version >= '3.8'
 fsspec[http]==2023.6.0 ; python_version >= '3.8'
-future==0.18.3 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+future==0.18.3 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
 gitdb==4.0.10 ; python_version >= '3.7'
-gitpython==3.1.37 ; python_version >= '3.7'
+gitpython==3.1.40 ; python_version >= '3.7'
 gputil==1.4.0
-greenlet==3.0.0 ; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
+greenlet==3.0.0 ; python_version >= '3' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))))
 h11==0.14.0 ; python_version >= '3.7'
 h2o-wave==0.26.2
 hf-transfer==0.1.3
+hjson==3.1.0
 httpcore==0.18.0 ; python_version >= '3.8'
 httpx==0.25.0 ; python_version >= '3.8'
 huggingface-hub==0.17.2
@@ -65,7 +67,7 @@ jsonschema==4.19.1 ; python_version >= '3.8'
 jsonschema-specifications==2023.7.1 ; python_version >= '3.8'
 kaggle==1.5.16
 keyring==24.2.0
-lit==17.0.2
+lit==17.0.3
 markupsafe==2.1.3 ; python_version >= '3.7'
 monotonic==1.6
 more-itertools==10.1.0 ; python_version >= '3.8'
@@ -75,20 +77,23 @@ multidict==6.0.4 ; python_version >= '3.7'
 multiprocess==0.70.15 ; python_version >= '3.7'
 neptune==1.6.3
 networkx==3.1 ; python_version >= '3.8'
-numpy==1.26.0
+ninja==1.11.1.1
+numpy==1.26.1
 oauthlib==3.2.2 ; python_version >= '3.6'
 openai==0.28.1
 packaging==23.2 ; python_version >= '3.7'
 pandas==2.1.1
 peft==0.5.0
-pillow==10.0.1 ; python_version >= '3.8'
+pillow==10.1.0 ; python_version >= '3.8'
 portalocker==2.8.2 ; python_version >= '3.8'
 protobuf==3.20.3
-psutil==5.9.5 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+psutil==5.9.6 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+py-cpuinfo==9.0.0
 pyarrow==13.0.0
 pycparser==2.21
+pydantic==1.10.13 ; python_version >= '3.7'
 pyjwt==2.8.0 ; python_version >= '3.7'
-python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
 python-editor==1.0.4
 python-slugify==8.0.1 ; python_version >= '3.7'
 pytz==2023.3.post1
@@ -100,7 +105,7 @@ requests==2.31.0 ; python_version >= '3.7'
 requests-oauthlib==1.3.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 rfc3339-validator==0.1.4
 rfc3987==1.3.8
-rpds-py==0.10.4 ; python_version >= '3.8'
+rpds-py==0.10.6 ; python_version >= '3.8'
 s3transfer==0.7.0 ; python_version >= '3.7'
 sacrebleu==2.0.0
 safetensors==0.4.0 ; python_version >= '3.7'
@@ -109,8 +114,8 @@ scipy==1.11.3 ; python_version < '3.13' and python_version >= '3.9'
 secretstorage==3.3.3 ; sys_platform == 'linux'
 sentencepiece==0.1.99
 setuptools==68.2.2 ; python_version >= '3.8'
-simplejson==3.19.2 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'
-six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+simplejson==3.19.2 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2'
+six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
 smmap==5.0.1 ; python_version >= '3.7'
 sniffio==1.3.0 ; python_version >= '3.7'
 soupsieve==2.5 ; python_version >= '3.8'
@@ -136,12 +141,12 @@ types-python-dateutil==2.8.19.14
 typing-extensions==4.8.0 ; python_version >= '3.8'
 tzdata==2023.3 ; python_version >= '2'
 uri-template==1.3.0
-urllib3==1.26.17 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+urllib3==2.0.7 ; python_version >= '3.7'
 uvicorn==0.23.2 ; python_version >= '3.8'
 wcwidth==0.2.8
 webcolors==1.13
 webencodings==0.5.1
-websocket-client==1.6.3 ; python_version >= '3.8'
+websocket-client==1.6.4 ; python_version >= '3.8'
 xxhash==3.4.1 ; python_version >= '3.7'
 xyzservices==2023.10.0 ; python_version >= '3.8'
 yarl==1.9.2 ; python_version >= '3.7'
diff --git a/train.py b/train.py
index c0fc7cb73..b00a789af 100644
--- a/train.py
+++ b/train.py
@@ -15,6 +15,7 @@
 from distutils import util
 from typing import Any, Callable, Dict, Tuple
 
+import deepspeed
 import numpy as np
 import pandas as pd
 import torch
@@ -22,6 +23,7 @@
 from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler
 from torch.utils.data import DataLoader
 from tqdm import tqdm
+from transformers.deepspeed import HfDeepSpeedConfig
 
 from llm_studio.src.datasets.text_utils import get_tokenizer
 from llm_studio.src.loggers import MainLogger
@@ -50,6 +52,7 @@
 )
 from llm_studio.src.utils.modeling_utils import (
     check_disk_space,
+    get_ds_config,
     get_number_of_validation_epochs,
     get_optimizer,
     get_scheduler,
@@ -151,6 +154,9 @@ def run_eval(
 def run_train(
     cfg: Any,
     model: torch.nn.Module,
+    optimizer,
+    scheduler,
+    epoch_steps,
     train_dataloader,
     val_dataloader,
     val_df: pd.DataFrame,
@@ -172,11 +178,6 @@ def run_train(
         Last train batch
     """
 
-    epoch_steps = len(train_dataloader)
-
-    optimizer = get_optimizer(model=model, cfg=cfg)
-    scheduler = get_scheduler(cfg=cfg, optimizer=optimizer, epoch_steps=epoch_steps)
-
     scaler: GradScaler | ShardedGradScaler | None = None
     if cfg.environment.mixed_precision:
         if cfg.environment.use_fsdp:
@@ -216,8 +217,10 @@ def run_train(
         if cfg.environment._local_rank == 0:
             logger.info(f"Training Epoch: {epoch + 1} / {cfg.training.epochs}")
 
-        if cfg.environment._distributed and hasattr(
-            train_dataloader.sampler, "set_epoch"
+        if (
+            cfg.environment._distributed
+            and not cfg.environment.use_deepspeed
+            and hasattr(train_dataloader.sampler, "set_epoch")
         ):
             train_dataloader.sampler.set_epoch(epoch)  # type: ignore
 
@@ -280,7 +283,7 @@ def run_train(
                 loss = loss / cfg.training.grad_accumulation
 
             # Backward pass
-            if cfg.environment.mixed_precision:
+            if cfg.environment.mixed_precision and not cfg.environment.use_deepspeed:
                 scaler.scale(loss).backward()  # type: ignore
                 if itr % cfg.training.grad_accumulation == 0:
                     if cfg.training.gradient_clip > 0:
@@ -292,7 +295,10 @@ def run_train(
                     scaler.update()
                     optimizer.zero_grad(set_to_none=True)
             else:
-                loss.backward()
+                if cfg.environment.use_deepspeed:
+                    model.backward(loss)
+                else:
+                    loss.backward()
                 if itr % cfg.training.grad_accumulation == 0:
                     if cfg.training.gradient_clip > 0:
                         torch.nn.utils.clip_grad_norm_(
@@ -349,28 +355,30 @@ def run_train(
                 if cfg.training.evaluation_epochs == 1:
                     progress_bar.close()
 
+                # TODO: Move back after fixing slow generation of deepspeed.
+                if not cfg.training.save_best_checkpoint:
+                    checkpoint_path = cfg.output_directory
+                    if cfg.environment._local_rank == 0:
+                        logger.info(
+                            f"Saving last model checkpoint to {checkpoint_path}"
+                        )
+                    save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
+
                 val_loss, val_metric = run_eval(
                     cfg=cfg, model=model, val_dataloader=val_dataloader, val_df=val_df
                 )
-                if cfg.environment._local_rank == 0:
-                    if cfg.training.save_best_checkpoint:
-                        if objective_op(val_metric, best_val_metric):
-                            checkpoint_path = cfg.output_directory
+
+                if cfg.training.save_best_checkpoint:
+                    if objective_op(val_metric, best_val_metric):
+                        checkpoint_path = cfg.output_directory
+                        if cfg.environment._local_rank == 0:
                             logger.info(
                                 f"Saving best model checkpoint: "
                                 f"val_{cfg.prediction.metric} {best_val_metric:.5} -> "
                                 f"{val_metric:.5} to {checkpoint_path}"
                             )
-                            save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
-                            best_val_metric = val_metric
-                    else:
-                        checkpoint_path = cfg.output_directory
-                        logger.info(
-                            f"Saving last model checkpoint: "
-                            f"val_loss {val_loss:.5}, val_{cfg.prediction.metric} "
-                            f"{val_metric:.5} to {checkpoint_path}"
-                        )
                         save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
+                        best_val_metric = val_metric
 
                 model.train()
 
@@ -395,6 +403,9 @@ def run_train(
 def run_train_rlhf(
     cfg: Any,
     model: torch.nn.Module,
+    optimizer,
+    scheduler,
+    epoch_steps,
     train_dataloader,
     val_dataloader,
     val_df: pd.DataFrame,
@@ -416,11 +427,6 @@ def run_train_rlhf(
         Last train batch
     """
 
-    epoch_steps = len(train_dataloader)
-
-    optimizer = get_optimizer(model=model, cfg=cfg)
-    scheduler = get_scheduler(cfg=cfg, optimizer=optimizer, epoch_steps=epoch_steps)
-
     scaler: GradScaler | ShardedGradScaler | None = None
     if cfg.environment.mixed_precision:
         if cfg.environment.use_fsdp:
@@ -563,6 +569,7 @@ def run_train_rlhf(
                         batch["reward_model_prompt_text"],
                         output_dict["predicted_text"],
                     )
+
                 if cfg.training.offload_reward_model:
                     reward_model.to("cpu")
 
@@ -679,25 +686,27 @@ def run_train_rlhf(
                 val_loss, val_metric = run_eval(
                     cfg=cfg, model=model, val_dataloader=val_dataloader, val_df=val_df
                 )
-                if cfg.environment._local_rank == 0:
-                    if cfg.training.save_best_checkpoint:
-                        if objective_op(val_metric, best_val_metric):
-                            checkpoint_path = cfg.output_directory
+
+                if cfg.training.save_best_checkpoint:
+                    if objective_op(val_metric, best_val_metric):
+                        checkpoint_path = cfg.output_directory
+                        if cfg.environment._local_rank == 0:
                             logger.info(
                                 f"Saving best model checkpoint: "
                                 f"val_{cfg.prediction.metric} {best_val_metric:.5} -> "
                                 f"{val_metric:.5} to {checkpoint_path}"
                             )
-                            save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
-                            best_val_metric = val_metric
-                    else:
-                        checkpoint_path = cfg.output_directory
+                        save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
+                        best_val_metric = val_metric
+                else:
+                    checkpoint_path = cfg.output_directory
+                    if cfg.environment._local_rank == 0:
                         logger.info(
                             f"Saving last model checkpoint: "
                             f"val_loss {val_loss:.5}, val_{cfg.prediction.metric} "
                             f"{val_metric:.5} to {checkpoint_path}"
                         )
-                        save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
+                    save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
 
                 model.train()
 
@@ -740,6 +749,17 @@ def run(cfg: Any) -> None:
     else:
         cfg.environment._seed = cfg.environment.seed
 
+    if cfg.environment.use_deepspeed and cfg.environment.use_fsdp:
+        raise ValueError("Deepspeed and FSDP cannot be used at the same time.")
+    if (
+        cfg.architecture.backbone_dtype in ["int8", "int4"]
+        and cfg.environment.use_deepspeed
+    ):
+        raise ValueError(
+            f"Deepspeed do not support backbone type {cfg.architecture.backbone_dtype}."
+            + " Please set backbone type to float16 or bfloat16 for using deepspeed."
+        )
+
     # Prepare environment
     if "WORLD_SIZE" in os.environ:
         cfg.environment._distributed = int(os.environ["WORLD_SIZE"]) > 1
@@ -749,7 +769,10 @@ def run(cfg: Any) -> None:
     if cfg.environment._distributed:
         cfg.environment._local_rank = int(os.environ["LOCAL_RANK"])
         cfg.environment._device = "cuda:%d" % cfg.environment._local_rank
-        torch.distributed.init_process_group(backend="nccl", init_method="env://")
+        if cfg.environment.use_deepspeed:
+            deepspeed.init_distributed()
+        else:
+            torch.distributed.init_process_group(backend="nccl", init_method="env://")
         cfg.environment._cpu_comm = torch.distributed.new_group(backend="gloo")
 
         cfg.environment._world_size = torch.distributed.get_world_size()
@@ -826,11 +849,14 @@ def run(cfg: Any) -> None:
             * cfg.environment._world_size
         )
 
-    # Prepare model
+    # Prepare model and optimizer
+    if cfg.environment.use_deepspeed:
+        ds_config = get_ds_config(cfg)
+        # keep this object alive.
+        dschf = HfDeepSpeedConfig(ds_config)  # noqa: F841
     with torch.device(cfg.environment._device):
         model = cfg.architecture.model_class(cfg)
-        if cfg.environment._local_rank == 0:
-            check_disk_space(model, cfg.output_directory)
+        check_disk_space(model, cfg.output_directory, cfg.environment.use_deepspeed)
 
         # load model weights
         if cfg.architecture.pretrained_weights != "":
@@ -838,6 +864,10 @@ def run(cfg: Any) -> None:
             load_checkpoint(cfg, model, strict=cfg.training.epochs == -1)
     model.to(cfg.environment._device)
 
+    epoch_steps = len(train_dataloader)
+    optimizer = get_optimizer(model=model, cfg=cfg)
+    scheduler = get_scheduler(cfg=cfg, optimizer=optimizer, epoch_steps=epoch_steps)
+
     if getattr(cfg.architecture, "force_embedding_gradients"):
         for module in model.modules():
             if isinstance(module, torch.nn.Embedding):
@@ -846,13 +876,33 @@ def run(cfg: Any) -> None:
                     param.data = param.data.float()
 
     if cfg.environment._distributed:
-        model = wrap_model_distributed(model, cfg, cfg.environment.use_fsdp)
+        (
+            model,
+            optimizer,
+            train_dataloader,
+            val_dataloader,
+            scheduler,
+        ) = wrap_model_distributed(
+            model=model,
+            optimizer=optimizer,
+            lr_scheduler=scheduler,
+            train_dataloader=train_dataloader,
+            val_dataloader=val_dataloader,
+            cfg=cfg,
+        )
 
     if cfg.environment.compile_model:
-        if cfg.environment._distributed:
-            model.module.backbone = torch.compile(model.module.backbone)
+        # deepspeed do not support torch.compile
+        if cfg.environment.use_deepspeed:
+            logger.warning(
+                "Deepspeed is active, but it doesn't support torch.compile."
+                "Skipping compilation for this experiment."
+            )
         else:
-            model.backbone = torch.compile(model.backbone)
+            if cfg.environment._distributed:
+                model.module.backbone = torch.compile(model.module.backbone)
+            else:
+                model.backbone = torch.compile(model.backbone)
 
     # Force settings when saving best checkpoint
     if cfg.training.save_best_checkpoint:
@@ -896,6 +946,9 @@ def run(cfg: Any) -> None:
     val_loss, val_metric = train_function(
         cfg=cfg,
         model=model,
+        optimizer=optimizer,
+        scheduler=scheduler,
+        epoch_steps=epoch_steps,
         train_dataloader=train_dataloader,
         val_dataloader=val_dataloader,
         val_df=val_df,
@@ -907,18 +960,17 @@ def run(cfg: Any) -> None:
 
     experiment_path = f"{cfg.output_directory}"
 
-    if cfg.environment._local_rank == 0:
-        if cfg.training.epochs == 0:
-            checkpoint_path = cfg.output_directory
-            logger.info(
-                f"Saving last model checkpoint: "
-                f"val_loss {val_loss:.5}, val_{cfg.prediction.metric} "
-                f"{val_metric:.5} to {checkpoint_path}"
-            )
-            save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
+    if cfg.training.epochs == 0:
+        checkpoint_path = cfg.output_directory
+        logger.info(
+            f"Saving last model checkpoint: "
+            f"val_loss {val_loss:.5}, val_{cfg.prediction.metric} "
+            f"{val_metric:.5} to {checkpoint_path}"
+        )
+        save_checkpoint(model=model, path=checkpoint_path, cfg=cfg)
 
+    if cfg.environment._local_rank == 0:
         save_config_yaml(f"{cfg.output_directory}/cfg.yaml", cfg)
-
         save_prediction_outputs(cfg.experiment_name, experiment_path)
 
         flag_path = os.path.join(cfg.output_directory, "flags.json")