From 7705822f05127666570c6cf0569aee87258e3e85 Mon Sep 17 00:00:00 2001 From: Daniel Peebles Date: Mon, 16 Nov 2015 15:50:46 +0100 Subject: [PATCH 01/20] WIP --- src/libexpr/eval.cc | 88 +++++++++++++++++++++++++++++++++++++++--- src/libexpr/eval.hh | 29 +++++++++++++- src/libexpr/primops.cc | 12 +++--- src/libexpr/value.hh | 4 +- 4 files changed, 119 insertions(+), 14 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 3b21c078fd5..aa902570cef 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -5,7 +5,10 @@ #include "derivations.hh" #include "globals.hh" #include "eval-inline.hh" +#include "value-to-json.hh" +#include +#include #include #include #include @@ -158,7 +161,8 @@ string showType(const Value & v) case tApp: return "a function application"; case tLambda: return "a function"; case tBlackhole: return "a black hole"; - case tPrimOp: return "a built-in function"; + case tPrimOp: + case tPrimOpLambda: return "a built-in function"; case tPrimOpApp: return "a partially applied built-in function"; case tExternal: return v.external->showType(); } @@ -262,6 +266,7 @@ EvalState::EvalState(const Strings & _searchPath) , sColumn(symbols.create("column")) , sFunctor(symbols.create("__functor")) , sToString(symbols.create("__toString")) + , evalMode(Record) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) { @@ -342,6 +347,50 @@ void EvalState::addPrimOp(const string & name, baseEnv.values[0]->attrs->push_back(Attr(sym, v)); } +void EvalState::addPrimOpLambda(const string & name, + unsigned int arity, PrimOpLambdaFun primOp) +{ + Value * v = allocValue(); + string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + Symbol sym = symbols.create(name2); + v->type = tPrimOpLambda; + v->primOpLambda = NEW PrimOpLambda(primOp, arity, sym); + staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; + baseEnv.values[baseEnvDispl++] = v; + baseEnv.values[0]->attrs->push_back(Attr(sym, v)); +} + + +std::string EvalState::valueToJSON(Value & value) { + std::ostringstream out; + PathSet context; + printValueAsJSON(*this, true, value, out, context); + return out.str(); +} + +void EvalState::addImpurePrimOp(const string & name, + unsigned int arity, PrimOpFun primOp) +{ + if (evalMode == Record) { + PrimOpLambdaFun foo = [this, name, primOp, arity] (EvalState & state, const Pos & pos, Value * * args, Value & v) { + std::cout << "WE'RE GETTING CALLED INSIDE " << name << std::endl; + std::list argList; + for (int i = 0; i < arity; i++) { + argList.push_back(valueToJSON(*args[i])); + } + primOp(state, pos, args, v); + recording[std::make_pair(name, argList)] = v; + }; + std::cout << "ZOMG " << name << std::endl; + addPrimOpLambda(name, arity, foo); + } else if (evalMode == Playback) { + std::cout << "ZOMG PLAYBACK " << name << std::endl; + } else { + addPrimOp(name, arity, primOp); + } +} + + void EvalState::getBuiltin(const string & name, Value & v) { @@ -622,6 +671,16 @@ void EvalState::resetFileCache() void EvalState::eval(Expr * e, Value & v) { e->eval(*this, baseEnv, v); + + std::fstream out(getEnv("NIX_RECORDING"), std::fstream::out); + for (auto kv : recording) { + out << kv.first.first << "("; + for (auto arg : kv.first.second) { + out << ", " << arg; + } + out << ") = " << kv.second << std::endl; + } + out.close(); } @@ -908,8 +967,19 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) argsDone++; primOp = primOp->primOpApp.left; } - assert(primOp->type == tPrimOp); - unsigned int arity = primOp->primOp->arity; + + unsigned int arity; + Symbol name; + + if (primOp->type == tPrimOp) { + arity = primOp->primOp->arity; + name = primOp->primOp->name; + } else if (primOp->type == tPrimOpLambda) { + arity = primOp->primOpLambda->arity; + name = primOp->primOpLambda->name; + } else { + abort(); + } unsigned int argsLeft = arity - argsDone; if (argsLeft == 1) { @@ -924,8 +994,13 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) /* And call the primop. */ nrPrimOpCalls++; - if (countCalls) primOpCalls[primOp->primOp->name]++; - primOp->primOp->fun(*this, pos, vArgs, v); + if (countCalls) primOpCalls[name]++; + + if (primOp->type == tPrimOp) { + primOp->primOp->fun(*this, pos, vArgs, v); + } else if (primOp->type == tPrimOpLambda) { + primOp->primOpLambda->fun(*this, pos, vArgs, v); + } } else { Value * fun2 = allocValue(); *fun2 = fun; @@ -938,7 +1013,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) { - if (fun.type == tPrimOp || fun.type == tPrimOpApp) { + if (fun.type == tPrimOp || fun.type == tPrimOpLambda || fun.type == tPrimOpApp) { callPrimOp(fun, arg, v, pos); return; } @@ -1432,6 +1507,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, } +// TODO: record & playback string EvalState::copyPathToStore(PathSet & context, const Path & path) { if (nix::isDerivation(path)) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 2d7b7bcdcb6..b2ff7ac29bf 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -18,9 +18,14 @@ namespace nix { class EvalState; +typedef enum { + Normal, + Record, + Playback +} EvalMode; typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); - +typedef std::function PrimOpLambdaFun; struct PrimOp { @@ -31,6 +36,14 @@ struct PrimOp : fun(fun), arity(arity), name(name) { } }; +struct PrimOpLambda +{ + PrimOpLambdaFun fun; + unsigned int arity; + Symbol name; + PrimOpLambda(PrimOpLambdaFun fun, unsigned int arity, Symbol name) + : fun(fun), arity(arity), name(name) { } +}; struct Env { @@ -95,6 +108,10 @@ private: SearchPath searchPath; + EvalMode evalMode; + + std::map>, Value> recording; + public: EvalState(const Strings & _searchPath); @@ -192,6 +209,16 @@ private: void addPrimOp(const string & name, unsigned int arity, PrimOpFun primOp); + void addPrimOpLambda(const string & name, + unsigned int arity, PrimOpLambdaFun primOp); + + + + std::string valueToJSON(Value & value); + + void addImpurePrimOp(const string & name, + unsigned int arity, PrimOpFun primOp); + public: void getBuiltin(const string & name, Value & v); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 87ee4f68a99..01e2dc72f71 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1770,14 +1770,14 @@ void EvalState::createBaseEnv() addPrimOp("__valueSize", 1, prim_valueSize); // Paths - addPrimOp("__toPath", 1, prim_toPath); - addPrimOp("__storePath", 1, prim_storePath); - addPrimOp("__pathExists", 1, prim_pathExists); + addImpurePrimOp("__toPath", 1, prim_toPath); + addImpurePrimOp("__storePath", 1, prim_storePath); + addImpurePrimOp("__pathExists", 1, prim_pathExists); addPrimOp("baseNameOf", 1, prim_baseNameOf); addPrimOp("dirOf", 1, prim_dirOf); - addPrimOp("__readFile", 1, prim_readFile); - addPrimOp("__readDir", 1, prim_readDir); - addPrimOp("__findFile", 2, prim_findFile); + addImpurePrimOp("__readFile", 1, prim_readFile); + addImpurePrimOp("__readDir", 1, prim_readDir); + addImpurePrimOp("__findFile", 2, prim_findFile); // Creating files addPrimOp("__toXML", 1, prim_toXML); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index e6d1502cb60..3e57f5889fc 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -20,6 +20,7 @@ typedef enum { tLambda, tBlackhole, tPrimOp, + tPrimOpLambda, tPrimOpApp, tExternal, } ValueType; @@ -30,7 +31,7 @@ struct Env; struct Expr; struct ExprLambda; struct PrimOp; -struct PrimOp; +struct PrimOpLambda; class Symbol; struct Pos; class EvalState; @@ -137,6 +138,7 @@ struct Value ExprLambda * fun; } lambda; PrimOp * primOp; + PrimOpLambda * primOpLambda; struct { Value * left, * right; } primOpApp; From f3df86a5f74d311264c84bb887b5de028274c730 Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Mon, 16 Nov 2015 17:36:30 +0000 Subject: [PATCH 02/20] WIP2 --- src/libexpr/eval.cc | 133 +++++++++++++++++++++++++++++++++++------ src/libexpr/eval.hh | 2 + src/libexpr/primops.cc | 6 +- 3 files changed, 121 insertions(+), 20 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index aa902570cef..8a607f712c7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -6,6 +6,8 @@ #include "globals.hh" #include "eval-inline.hh" #include "value-to-json.hh" +#include "json-to-value.hh" +#include #include #include @@ -266,7 +268,7 @@ EvalState::EvalState(const Strings & _searchPath) , sColumn(symbols.create("column")) , sFunctor(symbols.create("__functor")) , sToString(symbols.create("__toString")) - , evalMode(Record) + , evalMode(Normal) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) { @@ -285,8 +287,24 @@ EvalState::EvalState(const Strings & _searchPath) clearValue(vEmptySet); vEmptySet.type = tAttrs; vEmptySet.attrs = allocBindings(0); + + const char * recordMode = getenv("NIX_RECORDING"); + const char * playbackMode = getenv("NIX_PLAYBACK"); + if (recordMode && !playbackMode) { + evalMode = Record; + recordFileName = recordMode; + } else if (playbackMode && !recordMode) { + evalMode = Playback; + recordFileName = playbackMode; + } else if (!playbackMode && !recordMode) { + evalMode = Normal; + } + else + throw EvalError("can't use both NIX_RECORDING and NIX_PLAYBACK"); + std::cout << "EvalMode: " << evalMode << std::endl; createBaseEnv(); + } @@ -371,27 +389,42 @@ std::string EvalState::valueToJSON(Value & value) { void EvalState::addImpurePrimOp(const string & name, unsigned int arity, PrimOpFun primOp) { + std::cout << "IMPURE " << name << std::endl; + if (evalMode == Record) { PrimOpLambdaFun foo = [this, name, primOp, arity] (EvalState & state, const Pos & pos, Value * * args, Value & v) { - std::cout << "WE'RE GETTING CALLED INSIDE " << name << std::endl; std::list argList; for (int i = 0; i < arity; i++) { argList.push_back(valueToJSON(*args[i])); } + std::cout << "Record " << name << std::endl; primOp(state, pos, args, v); recording[std::make_pair(name, argList)] = v; }; - std::cout << "ZOMG " << name << std::endl; addPrimOpLambda(name, arity, foo); } else if (evalMode == Playback) { - std::cout << "ZOMG PLAYBACK " << name << std::endl; + PrimOpLambdaFun foo = [this, name, primOp, arity] (EvalState & state, const Pos & pos, Value * * args, Value & v) { + std::list argList; + for (int i = 0; i < arity; i++) { + argList.push_back(valueToJSON(*args[i])); + } + std::cout << "Playback " << name << std::endl; + auto result = recording.find(std::make_pair(name, argList)); + if (result == recording.end()) { + std::string errorMsg("wanted to call "); + errorMsg +=name; + throw EvalError(errorMsg.c_str()); + } + else { + v = result->second; + } + }; + addPrimOpLambda(name, arity, foo); } else { addPrimOp(name, arity, primOp); } } - - void EvalState::getBuiltin(const string & name, Value & v) { v = *baseEnv.values[0]->attrs->find(symbols.create(name))->value; @@ -668,19 +701,83 @@ void EvalState::resetFileCache() } +void EvalState::getAttr(Value & attrSet, const char *attr, Value & v) { + forceAttrs(attrSet); + // !!! Should we create a symbol here or just do a lookup? + Bindings::iterator i = attrSet.attrs->find(symbols.create(attr)); + if (i == attrSet.attrs->end()) + throwEvalError("attribute ‘%1%’ missing", attr); + // !!! add to stack trace? + forceValue(*i->value); + v = *i->value; +} + void EvalState::eval(Expr * e, Value & v) -{ +{ + if (evalMode == Playback) { + Value top; + std::fstream in(recordFileName, std::fstream::in); + std::stringstream buffer; + buffer << in.rdbuf(); + parseJSON(*this, buffer.str(), top); + Value functions; + getAttr(top, "functions", functions); + forceList(functions); + for (unsigned int i = 0; i < functions.listSize(); ++i) { + Value ¤t = *functions.listElems()[i]; + Value name, parameters, result; + getAttr(current, "name", name); + getAttr(current, "parameters", parameters); + getAttr(current, "result", result); + std::string nameString = forceStringNoCtx(name); + std::list parameterList; + forceList(parameters); + for (unsigned int j = 0; j < parameters.listSize(); ++j) { + parameterList.push_back(valueToJSON(*parameters.listElems()[j])); + } + recording[std::make_pair(nameString, parameterList)] = result; + } + Value sources; + getAttr(top, "sources", sources); + forceAttrs(sources); + for (auto it = sources.attrs->begin(); it != sources.attrs->end(); ++it) { + srcToStore[it->name] = forceStringNoCtx(*it->value); + } + } + e->eval(*this, baseEnv, v); - - std::fstream out(getEnv("NIX_RECORDING"), std::fstream::out); - for (auto kv : recording) { - out << kv.first.first << "("; - for (auto arg : kv.first.second) { - out << ", " << arg; - } - out << ") = " << kv.second << std::endl; + if (evalMode == Record) { + std::fstream out(recordFileName, std::fstream::out); + out << "{\"functions\": [\n"; + bool isThisTheFirstTime = true; + + for (auto kv : recording) { + std::cout << "?"; + if (!isThisTheFirstTime) out << ","; + isThisTheFirstTime = false; + + out << "{ \"name\": \"" << kv.first.first << "\", \"parameters\": ["; + + bool isThisTheFirstTime2 = true; + for (auto parameter: kv.first.second) { + if (!isThisTheFirstTime2) out << ", "; + isThisTheFirstTime2 = false; + out << parameter; + } + out << "], \"result\": " << valueToJSON(kv.second) << "}\n"; + } + + out << "], \"sources\": {"; + + bool isThisTheFirstTime3 = true; + for (auto path : srcToStore) { + if (!isThisTheFirstTime3) out << ", "; + out << "\"" << path.first << "\": \"" << path.second << "\""; + } + std::cout << "\\"; + out << "}}"; + out.close(); } - out.close(); } @@ -1507,7 +1604,6 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, } -// TODO: record & playback string EvalState::copyPathToStore(PathSet & context, const Path & path) { if (nix::isDerivation(path)) @@ -1517,6 +1613,9 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path) if (srcToStore[path] != "") dstPath = srcToStore[path]; else { + if (evalMode == Playback) { + throwEvalError("Unknown path encountered in playback mode: '%1%'", path); + } dstPath = settings.readOnlyMode ? computeStorePathForPath(checkSourcePath(path)).first : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index b2ff7ac29bf..35ed83cc1b8 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -109,6 +109,7 @@ private: SearchPath searchPath; EvalMode evalMode; + const char * recordFileName; std::map>, Value> recording; @@ -298,6 +299,7 @@ private: friend struct ExprOpConcatLists; friend struct ExprSelect; friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); + void getAttr(Value & top, const char* arg2, Value & v); }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 01e2dc72f71..49c3b1be152 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1747,7 +1747,7 @@ void EvalState::createBaseEnv() forceValue(v); addConstant("import", v); if (settings.enableImportNative) - addPrimOp("__importNative", 2, prim_importNative); + addImpurePrimOp("__importNative", 2, prim_importNative); addPrimOp("__typeOf", 1, prim_typeOf); addPrimOp("isNull", 1, prim_isNull); addPrimOp("__isFunction", 1, prim_isFunction); @@ -1759,7 +1759,7 @@ void EvalState::createBaseEnv() addPrimOp("throw", 1, prim_throw); addPrimOp("__addErrorContext", 2, prim_addErrorContext); addPrimOp("__tryEval", 1, prim_tryEval); - addPrimOp("__getEnv", 1, prim_getEnv); + addImpurePrimOp("__getEnv", 1, prim_getEnv); // Strictness addPrimOp("__seq", 2, prim_seq); @@ -1770,7 +1770,7 @@ void EvalState::createBaseEnv() addPrimOp("__valueSize", 1, prim_valueSize); // Paths - addImpurePrimOp("__toPath", 1, prim_toPath); + addPrimOp("__toPath", 1, prim_toPath); addImpurePrimOp("__storePath", 1, prim_storePath); addImpurePrimOp("__pathExists", 1, prim_pathExists); addPrimOp("baseNameOf", 1, prim_baseNameOf); From 239043f5c31cb47037f78cda1fd62e3eab470455 Mon Sep 17 00:00:00 2001 From: Daniel Peebles Date: Tue, 17 Nov 2015 10:40:27 +0100 Subject: [PATCH 03/20] MOAR WIP --- src/libexpr/eval.cc | 72 ++++++++++++++++++++++++-------------------- src/libexpr/eval.hh | 2 +- src/libexpr/parser.y | 12 +++++++- 3 files changed, 51 insertions(+), 35 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 8a607f712c7..e4f9dd256b4 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -296,6 +296,41 @@ EvalState::EvalState(const Strings & _searchPath) } else if (playbackMode && !recordMode) { evalMode = Playback; recordFileName = playbackMode; + + std::cout << "ZOMG PLAYBACK !!!!" << std::endl;; + Value top; + std::fstream in(recordFileName, std::fstream::in); + std::stringstream buffer; + buffer << in.rdbuf(); + parseJSON(*this, buffer.str(), top); + Value functions; + getAttr(top, "functions", functions); + forceList(functions); + for (unsigned int i = 0; i < functions.listSize(); ++i) { + Value ¤t = *functions.listElems()[i]; + Value name, parameters, result; + getAttr(current, "name", name); + getAttr(current, "parameters", parameters); + getAttr(current, "result", result); + std::string nameString = forceStringNoCtx(name); + std::list parameterList; + forceList(parameters); + for (unsigned int j = 0; j < parameters.listSize(); ++j) { + parameterList.push_back(valueToJSON(*parameters.listElems()[j])); + } + recording[std::make_pair(nameString, parameterList)] = result; + } + Value sources; + getAttr(top, "sources", sources); + forceAttrs(sources); + std::cout << sources.attrs->size() << std::endl; + for (auto it = sources.attrs->begin(); it != sources.attrs->end(); ++it) { + std::cout << it->name << " -> " << forceStringNoCtx(*it->value) << std::endl; + srcToStore[it->name] = forceStringNoCtx(*it->value); + } + + + } else if (!playbackMode && !recordMode) { evalMode = Normal; } @@ -714,38 +749,8 @@ void EvalState::getAttr(Value & attrSet, const char *attr, Value & v) { void EvalState::eval(Expr * e, Value & v) { - if (evalMode == Playback) { - Value top; - std::fstream in(recordFileName, std::fstream::in); - std::stringstream buffer; - buffer << in.rdbuf(); - parseJSON(*this, buffer.str(), top); - Value functions; - getAttr(top, "functions", functions); - forceList(functions); - for (unsigned int i = 0; i < functions.listSize(); ++i) { - Value ¤t = *functions.listElems()[i]; - Value name, parameters, result; - getAttr(current, "name", name); - getAttr(current, "parameters", parameters); - getAttr(current, "result", result); - std::string nameString = forceStringNoCtx(name); - std::list parameterList; - forceList(parameters); - for (unsigned int j = 0; j < parameters.listSize(); ++j) { - parameterList.push_back(valueToJSON(*parameters.listElems()[j])); - } - recording[std::make_pair(nameString, parameterList)] = result; - } - Value sources; - getAttr(top, "sources", sources); - forceAttrs(sources); - for (auto it = sources.attrs->begin(); it != sources.attrs->end(); ++it) { - srcToStore[it->name] = forceStringNoCtx(*it->value); - } - } - e->eval(*this, baseEnv, v); + if (evalMode == Record) { std::fstream out(recordFileName, std::fstream::out); out << "{\"functions\": [\n"; @@ -772,6 +777,7 @@ void EvalState::eval(Expr * e, Value & v) bool isThisTheFirstTime3 = true; for (auto path : srcToStore) { if (!isThisTheFirstTime3) out << ", "; + isThisTheFirstTime3 = false; out << "\"" << path.first << "\": \"" << path.second << "\""; } std::cout << "\\"; @@ -1604,7 +1610,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, } -string EvalState::copyPathToStore(PathSet & context, const Path & path) +string EvalState::copyPathToStore(PathSet & context, const Path & path, bool ignoreReadOnly) { if (nix::isDerivation(path)) throwEvalError("file names are not allowed to end in ‘%1%’", drvExtension); @@ -1616,7 +1622,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path) if (evalMode == Playback) { throwEvalError("Unknown path encountered in playback mode: '%1%'", path); } - dstPath = settings.readOnlyMode + dstPath = (settings.readOnlyMode && !ignoreReadOnly) ? computeStorePathForPath(checkSourcePath(path)).first : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); srcToStore[path] = dstPath; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 35ed83cc1b8..abc716e0c16 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -183,7 +183,7 @@ public: string coerceToString(const Pos & pos, Value & v, PathSet & context, bool coerceMore = false, bool copyToStore = true); - string copyPathToStore(PathSet & context, const Path & path); + string copyPathToStore(PathSet & context, const Path & path, bool ignoreReadOnly = false); /* Path coercion. Converts strings, paths and derivations to a path. The result is guaranteed to be a canonicalised, absolute diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index d34882f361c..7791ac10ca5 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -21,6 +21,8 @@ #include "nixexpr.hh" #include "eval.hh" +#include + namespace nix { struct ParseData @@ -574,7 +576,15 @@ Expr * EvalState::parseExprFromFile(const Path & path) Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv) { - return parse(readFile(path).c_str(), path, dirOf(path), staticEnv); + Path actualPath; + if (evalMode == Record || evalMode == Playback) { + PathSet context; + actualPath = copyPathToStore(context, path, true); + std::cout << "parseExprFromFile(" << path << ") = " << actualPath << "\n"; + } else { + actualPath = path; + } + return parse(readFile(actualPath).c_str(), path, dirOf(path), staticEnv); } From d0bda8f77a5977a1ad678c8634c55fe2bf01b9a4 Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Tue, 17 Nov 2015 13:56:47 +0000 Subject: [PATCH 04/20] working version --- src/libexpr/eval.cc | 38 --------------------------- src/libexpr/eval.hh | 59 +++++++++++++++++++++++++++++++++++++++--- src/libexpr/primops.cc | 24 ++++++++++++----- src/libexpr/value.hh | 5 +++- 4 files changed, 77 insertions(+), 49 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e4f9dd256b4..f77d8dac16f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -421,44 +421,6 @@ std::string EvalState::valueToJSON(Value & value) { return out.str(); } -void EvalState::addImpurePrimOp(const string & name, - unsigned int arity, PrimOpFun primOp) -{ - std::cout << "IMPURE " << name << std::endl; - - if (evalMode == Record) { - PrimOpLambdaFun foo = [this, name, primOp, arity] (EvalState & state, const Pos & pos, Value * * args, Value & v) { - std::list argList; - for (int i = 0; i < arity; i++) { - argList.push_back(valueToJSON(*args[i])); - } - std::cout << "Record " << name << std::endl; - primOp(state, pos, args, v); - recording[std::make_pair(name, argList)] = v; - }; - addPrimOpLambda(name, arity, foo); - } else if (evalMode == Playback) { - PrimOpLambdaFun foo = [this, name, primOp, arity] (EvalState & state, const Pos & pos, Value * * args, Value & v) { - std::list argList; - for (int i = 0; i < arity; i++) { - argList.push_back(valueToJSON(*args[i])); - } - std::cout << "Playback " << name << std::endl; - auto result = recording.find(std::make_pair(name, argList)); - if (result == recording.end()) { - std::string errorMsg("wanted to call "); - errorMsg +=name; - throw EvalError(errorMsg.c_str()); - } - else { - v = result->second; - } - }; - addPrimOpLambda(name, arity, foo); - } else { - addPrimOp(name, arity, primOp); - } -} void EvalState::getBuiltin(const string & name, Value & v) { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index abc716e0c16..4f60fdc588e 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -5,6 +5,7 @@ #include "nixexpr.hh" #include "symbol-table.hh" #include "hash.hh" +#include #include @@ -217,9 +218,61 @@ private: std::string valueToJSON(Value & value); - void addImpurePrimOp(const string & name, - unsigned int arity, PrimOpFun primOp); - + + template< PrimOpFun primOp, const char *name, unsigned int arity> + class WrapPrimOp + { + public: + static void recordPrimOp(EvalState & state, const Pos & pos, Value * * args, Value & v) + { + std::list argList; + for (int i = 0; i < arity; i++) { + argList.push_back(state.valueToJSON(*args[i])); + } + std::cout << "Record " << name << std::endl; + primOp(state, pos, args, v); + state.recording[std::make_pair(name, argList)] = v; + } + + static void playbackPrimOp(EvalState & state, const Pos & pos, Value * * args, Value & v) + { + std::list argList; + for (int i = 0; i < arity; i++) { + argList.push_back(state.valueToJSON(*args[i])); + } + std::cout << "Playback " << name << std::endl; + auto result = state.recording.find(std::make_pair(name, argList)); + if (result == state.recording.end()) { + std::string errorMsg("wanted to call "); + errorMsg +=name; + throw EvalError(errorMsg.c_str()); + } + else { + v = result->second; + } + } + + }; + + template< const char *name, unsigned int arity, PrimOpFun primOp> + void addImpurePrimOp() + { + if (evalMode == Record) { + PrimOpFun wrapped = WrapPrimOp::recordPrimOp; + addPrimOp(name, arity, wrapped); + } else if (evalMode == Playback) { + PrimOpFun wrapped = WrapPrimOp::playbackPrimOp; + addPrimOp(name, arity, wrapped ); + } else { + addPrimOp(name, arity, primOp); + } + } + + + + + + public: void getBuiltin(const string & name, Value & v); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 49c3b1be152..a34847c003b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1699,6 +1699,16 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args * Primop registration *************************************************************/ +extern const char __importNative[] = "__importNative"; +extern const char __findFile[] = "__findFile"; +extern const char __readDir[] = "__readDir"; +extern const char __getEnv[] = "__getEnv"; +extern const char __storePath[] = "__storePath"; +extern const char __pathExists[] = "__pathExists"; +extern const char __readFile[] = "__readFile"; + + + void EvalState::createBaseEnv() { @@ -1747,7 +1757,7 @@ void EvalState::createBaseEnv() forceValue(v); addConstant("import", v); if (settings.enableImportNative) - addImpurePrimOp("__importNative", 2, prim_importNative); + addImpurePrimOp<__importNative, 2, prim_importNative>(); addPrimOp("__typeOf", 1, prim_typeOf); addPrimOp("isNull", 1, prim_isNull); addPrimOp("__isFunction", 1, prim_isFunction); @@ -1759,7 +1769,7 @@ void EvalState::createBaseEnv() addPrimOp("throw", 1, prim_throw); addPrimOp("__addErrorContext", 2, prim_addErrorContext); addPrimOp("__tryEval", 1, prim_tryEval); - addImpurePrimOp("__getEnv", 1, prim_getEnv); + addImpurePrimOp<__getEnv, 1, prim_getEnv>(); // Strictness addPrimOp("__seq", 2, prim_seq); @@ -1771,13 +1781,13 @@ void EvalState::createBaseEnv() // Paths addPrimOp("__toPath", 1, prim_toPath); - addImpurePrimOp("__storePath", 1, prim_storePath); - addImpurePrimOp("__pathExists", 1, prim_pathExists); + addImpurePrimOp<__storePath, 1, prim_storePath>(); + addImpurePrimOp<__pathExists, 1, prim_pathExists>(); addPrimOp("baseNameOf", 1, prim_baseNameOf); addPrimOp("dirOf", 1, prim_dirOf); - addImpurePrimOp("__readFile", 1, prim_readFile); - addImpurePrimOp("__readDir", 1, prim_readDir); - addImpurePrimOp("__findFile", 2, prim_findFile); + addImpurePrimOp<__readFile, 1, prim_readFile>(); + addImpurePrimOp<__readDir, 1, prim_readDir>(); + addImpurePrimOp<__findFile, 2, prim_findFile>(); // Creating files addPrimOp("__toXML", 1, prim_toXML); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 3e57f5889fc..28bf69d89c1 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -1,6 +1,7 @@ #pragma once #include "symbol-table.hh" +#include namespace nix { @@ -36,6 +37,7 @@ class Symbol; struct Pos; class EvalState; class XMLWriter; +struct ConstantWithSideEffect; typedef long NixInt; @@ -139,7 +141,8 @@ struct Value } lambda; PrimOp * primOp; PrimOpLambda * primOpLambda; - struct { + ConstantWithSideEffect * constantWithSideEffect; + struct { Value * left, * right; } primOpApp; ExternalValueBase * external; From f3b593e07ddd014648da50052d01a5a33ea42506 Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Tue, 17 Nov 2015 14:50:57 +0000 Subject: [PATCH 05/20] hey, the LAST bug fixed! --- src/libexpr/eval.cc | 72 ++++++++++++++++++++++----------------------- src/libexpr/eval.hh | 12 +++----- 2 files changed, 40 insertions(+), 44 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f77d8dac16f..1a355b29ab2 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -346,6 +346,40 @@ EvalState::EvalState(const Strings & _searchPath) EvalState::~EvalState() { fileEvalCache.clear(); + + if (evalMode == Record) { + std::fstream out(recordFileName, std::fstream::out); + out << "{\"functions\": [\n"; + bool isThisTheFirstTime = true; + + for (auto kv : recording) { + std::cout << "?"; + if (!isThisTheFirstTime) out << ","; + isThisTheFirstTime = false; + + out << "{ \"name\": \"" << kv.first.first << "\", \"parameters\": ["; + + bool isThisTheFirstTime2 = true; + for (auto parameter: kv.first.second) { + if (!isThisTheFirstTime2) out << ", "; + isThisTheFirstTime2 = false; + out << parameter; + } + out << "], \"result\": " << valueToJSON(kv.second) << "}\n"; + } + + out << "], \"sources\": {"; + + bool isThisTheFirstTime3 = true; + for (auto path : srcToStore) { + if (!isThisTheFirstTime3) out << ", "; + isThisTheFirstTime3 = false; + out << "\"" << path.first << "\": \"" << path.second << "\""; + } + std::cout << "\\"; + out << "}}"; + out.close(); + } } @@ -709,43 +743,9 @@ void EvalState::getAttr(Value & attrSet, const char *attr, Value & v) { v = *i->value; } -void EvalState::eval(Expr * e, Value & v) -{ +void EvalState::eval(Expr* e, Value& v) +{ e->eval(*this, baseEnv, v); - - if (evalMode == Record) { - std::fstream out(recordFileName, std::fstream::out); - out << "{\"functions\": [\n"; - bool isThisTheFirstTime = true; - - for (auto kv : recording) { - std::cout << "?"; - if (!isThisTheFirstTime) out << ","; - isThisTheFirstTime = false; - - out << "{ \"name\": \"" << kv.first.first << "\", \"parameters\": ["; - - bool isThisTheFirstTime2 = true; - for (auto parameter: kv.first.second) { - if (!isThisTheFirstTime2) out << ", "; - isThisTheFirstTime2 = false; - out << parameter; - } - out << "], \"result\": " << valueToJSON(kv.second) << "}\n"; - } - - out << "], \"sources\": {"; - - bool isThisTheFirstTime3 = true; - for (auto path : srcToStore) { - if (!isThisTheFirstTime3) out << ", "; - isThisTheFirstTime3 = false; - out << "\"" << path.first << "\": \"" << path.second << "\""; - } - std::cout << "\\"; - out << "}}"; - out.close(); - } } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 4f60fdc588e..4f1128a526f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -220,9 +220,6 @@ private: template< PrimOpFun primOp, const char *name, unsigned int arity> - class WrapPrimOp - { - public: static void recordPrimOp(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::list argList; @@ -234,6 +231,7 @@ private: state.recording[std::make_pair(name, argList)] = v; } + template< PrimOpFun primOp, const char *name, unsigned int arity> static void playbackPrimOp(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::list argList; @@ -252,17 +250,15 @@ private: } } - }; - template< const char *name, unsigned int arity, PrimOpFun primOp> void addImpurePrimOp() { if (evalMode == Record) { - PrimOpFun wrapped = WrapPrimOp::recordPrimOp; + PrimOpFun wrapped = recordPrimOp; addPrimOp(name, arity, wrapped); } else if (evalMode == Playback) { - PrimOpFun wrapped = WrapPrimOp::playbackPrimOp; - addPrimOp(name, arity, wrapped ); + PrimOpFun wrapped = playbackPrimOp; + addPrimOp(name, arity, wrapped); } else { addPrimOp(name, arity, primOp); } From 946b76aaef039d1a677ba5754f9ff8990bb76e5b Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Wed, 18 Nov 2015 04:40:01 +0000 Subject: [PATCH 06/20] remove obsolete PrimOpLambda, remove some std::cout statements --- src/libexpr/eval.cc | 46 +++++++----------------------------------- src/libexpr/eval.hh | 24 +++------------------- src/libexpr/parser.y | 3 --- src/libexpr/primops.cc | 4 ++-- src/libexpr/value.hh | 8 +------- 5 files changed, 13 insertions(+), 72 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1a355b29ab2..379360bc6ac 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -163,8 +163,7 @@ string showType(const Value & v) case tApp: return "a function application"; case tLambda: return "a function"; case tBlackhole: return "a black hole"; - case tPrimOp: - case tPrimOpLambda: return "a built-in function"; + case tPrimOp: return "a built-in function"; case tPrimOpApp: return "a partially applied built-in function"; case tExternal: return v.external->showType(); } @@ -297,7 +296,6 @@ EvalState::EvalState(const Strings & _searchPath) evalMode = Playback; recordFileName = playbackMode; - std::cout << "ZOMG PLAYBACK !!!!" << std::endl;; Value top; std::fstream in(recordFileName, std::fstream::in); std::stringstream buffer; @@ -376,7 +374,6 @@ EvalState::~EvalState() isThisTheFirstTime3 = false; out << "\"" << path.first << "\": \"" << path.second << "\""; } - std::cout << "\\"; out << "}}"; out.close(); } @@ -434,19 +431,6 @@ void EvalState::addPrimOp(const string & name, baseEnv.values[0]->attrs->push_back(Attr(sym, v)); } -void EvalState::addPrimOpLambda(const string & name, - unsigned int arity, PrimOpLambdaFun primOp) -{ - Value * v = allocValue(); - string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; - Symbol sym = symbols.create(name2); - v->type = tPrimOpLambda; - v->primOpLambda = NEW PrimOpLambda(primOp, arity, sym); - staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; - baseEnv.values[baseEnvDispl++] = v; - baseEnv.values[0]->attrs->push_back(Attr(sym, v)); -} - std::string EvalState::valueToJSON(Value & value) { std::ostringstream out; @@ -732,7 +716,7 @@ void EvalState::resetFileCache() } -void EvalState::getAttr(Value & attrSet, const char *attr, Value & v) { +void EvalState::getAttr(Value & attrSet, const char * attr, Value & v) { forceAttrs(attrSet); // !!! Should we create a symbol here or just do a lookup? Bindings::iterator i = attrSet.attrs->find(symbols.create(attr)); @@ -1032,19 +1016,8 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) argsDone++; primOp = primOp->primOpApp.left; } - - unsigned int arity; - Symbol name; - - if (primOp->type == tPrimOp) { - arity = primOp->primOp->arity; - name = primOp->primOp->name; - } else if (primOp->type == tPrimOpLambda) { - arity = primOp->primOpLambda->arity; - name = primOp->primOpLambda->name; - } else { - abort(); - } + assert(primOp->type == tPrimOp); + unsigned int arity = primOp->primOp->arity; unsigned int argsLeft = arity - argsDone; if (argsLeft == 1) { @@ -1059,13 +1032,8 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) /* And call the primop. */ nrPrimOpCalls++; - if (countCalls) primOpCalls[name]++; - - if (primOp->type == tPrimOp) { - primOp->primOp->fun(*this, pos, vArgs, v); - } else if (primOp->type == tPrimOpLambda) { - primOp->primOpLambda->fun(*this, pos, vArgs, v); - } + if (countCalls) primOpCalls[primOp->primOp->name]++; + primOp->primOp->fun(*this, pos, vArgs, v); } else { Value * fun2 = allocValue(); *fun2 = fun; @@ -1078,7 +1046,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) { - if (fun.type == tPrimOp || fun.type == tPrimOpLambda || fun.type == tPrimOpApp) { + if (fun.type == tPrimOp || fun.type == tPrimOpApp) { callPrimOp(fun, arg, v, pos); return; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 4f1128a526f..0cedf4ed791 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -26,7 +26,7 @@ typedef enum { } EvalMode; typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); -typedef std::function PrimOpLambdaFun; + struct PrimOp { @@ -37,14 +37,6 @@ struct PrimOp : fun(fun), arity(arity), name(name) { } }; -struct PrimOpLambda -{ - PrimOpLambdaFun fun; - unsigned int arity; - Symbol name; - PrimOpLambda(PrimOpLambdaFun fun, unsigned int arity, Symbol name) - : fun(fun), arity(arity), name(name) { } -}; struct Env { @@ -112,6 +104,7 @@ private: EvalMode evalMode; const char * recordFileName; + //TODO: use some other structure, maybe a hashmap std::map>, Value> recording; public: @@ -211,14 +204,8 @@ private: void addPrimOp(const string & name, unsigned int arity, PrimOpFun primOp); - void addPrimOpLambda(const string & name, - unsigned int arity, PrimOpLambdaFun primOp); - - - std::string valueToJSON(Value & value); - - + template< PrimOpFun primOp, const char *name, unsigned int arity> static void recordPrimOp(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -263,11 +250,6 @@ private: addPrimOp(name, arity, primOp); } } - - - - - public: diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 7791ac10ca5..7813fe943d5 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -21,8 +21,6 @@ #include "nixexpr.hh" #include "eval.hh" -#include - namespace nix { struct ParseData @@ -580,7 +578,6 @@ Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv) if (evalMode == Record || evalMode == Playback) { PathSet context; actualPath = copyPathToStore(context, path, true); - std::cout << "parseExprFromFile(" << path << ") = " << actualPath << "\n"; } else { actualPath = path; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a34847c003b..72ca15aff17 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1708,8 +1708,6 @@ extern const char __pathExists[] = "__pathExists"; extern const char __readFile[] = "__readFile"; - - void EvalState::createBaseEnv() { baseEnv.up = 0; @@ -1794,6 +1792,8 @@ void EvalState::createBaseEnv() addPrimOp("__toJSON", 1, prim_toJSON); addPrimOp("__fromJSON", 1, prim_fromJSON); addPrimOp("__toFile", 2, prim_toFile); + + //TODO: filter source is an impure primop too addPrimOp("__filterSource", 2, prim_filterSource); // Sets diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 28bf69d89c1..38c9da1f1f7 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -1,7 +1,6 @@ #pragma once #include "symbol-table.hh" -#include namespace nix { @@ -21,7 +20,6 @@ typedef enum { tLambda, tBlackhole, tPrimOp, - tPrimOpLambda, tPrimOpApp, tExternal, } ValueType; @@ -32,12 +30,10 @@ struct Env; struct Expr; struct ExprLambda; struct PrimOp; -struct PrimOpLambda; class Symbol; struct Pos; class EvalState; class XMLWriter; -struct ConstantWithSideEffect; typedef long NixInt; @@ -140,9 +136,7 @@ struct Value ExprLambda * fun; } lambda; PrimOp * primOp; - PrimOpLambda * primOpLambda; - ConstantWithSideEffect * constantWithSideEffect; - struct { + struct { Value * left, * right; } primOpApp; ExternalValueBase * external; From e2498d9b45a3db5ac6c6afeb2559971983fd545d Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Wed, 18 Nov 2015 23:35:23 +0000 Subject: [PATCH 07/20] cleaning up, capturing functionality in functions --- src/libexpr/eval.cc | 178 ++++++++++++++++++++++------------------- src/libexpr/eval.hh | 106 +++++++++++++----------- src/libexpr/parser.y | 6 +- src/libexpr/primops.cc | 9 +-- 4 files changed, 162 insertions(+), 137 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 379360bc6ac..1ea32566d8a 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -7,9 +7,9 @@ #include "eval-inline.hh" #include "value-to-json.hh" #include "json-to-value.hh" -#include #include +#include #include #include #include @@ -287,99 +287,112 @@ EvalState::EvalState(const Strings & _searchPath) vEmptySet.type = tAttrs; vEmptySet.attrs = allocBindings(0); + initializeDeterministicEvaluationMode(); + createBaseEnv(); +} + +void EvalState::initializeDeterministicEvaluationMode() +{ const char * recordMode = getenv("NIX_RECORDING"); const char * playbackMode = getenv("NIX_PLAYBACK"); if (recordMode && !playbackMode) { - evalMode = Record; - recordFileName = recordMode; + evalMode = Record; + recordFileName = recordMode; } else if (playbackMode && !recordMode) { - evalMode = Playback; - recordFileName = playbackMode; - - Value top; - std::fstream in(recordFileName, std::fstream::in); - std::stringstream buffer; - buffer << in.rdbuf(); - parseJSON(*this, buffer.str(), top); - Value functions; - getAttr(top, "functions", functions); - forceList(functions); - for (unsigned int i = 0; i < functions.listSize(); ++i) { - Value ¤t = *functions.listElems()[i]; - Value name, parameters, result; - getAttr(current, "name", name); - getAttr(current, "parameters", parameters); - getAttr(current, "result", result); - std::string nameString = forceStringNoCtx(name); - std::list parameterList; - forceList(parameters); - for (unsigned int j = 0; j < parameters.listSize(); ++j) { - parameterList.push_back(valueToJSON(*parameters.listElems()[j])); - } - recording[std::make_pair(nameString, parameterList)] = result; - } - Value sources; - getAttr(top, "sources", sources); - forceAttrs(sources); - std::cout << sources.attrs->size() << std::endl; - for (auto it = sources.attrs->begin(); it != sources.attrs->end(); ++it) { - std::cout << it->name << " -> " << forceStringNoCtx(*it->value) << std::endl; - srcToStore[it->name] = forceStringNoCtx(*it->value); - } - - - + evalMode = Playback; + recordFileName = playbackMode; + initializePlayback(); } else if (!playbackMode && !recordMode) { - evalMode = Normal; + evalMode = Normal; } else - throw EvalError("can't use both NIX_RECORDING and NIX_PLAYBACK"); - std::cout << "EvalMode: " << evalMode << std::endl; - - createBaseEnv(); - + throw EvalError("can't use both NIX_RECORDING and NIX_PLAYBACK"); + + if (evalMode == Record || evalMode == Playback) { + std::cerr << "Running in deterministic evaluation mode: " << evalMode << std::endl; + } } +void EvalState::initializePlayback() +{ + Symbol functionsSymbol(symbols.create("functions")), + nameSymbol(symbols.create("name")), + parametersSymbol(symbols.create("parameters")), + resultSymbol(symbols.create("result")), + sourcesSymbol(symbols.create("sources")); + Value top; + + parseJSON(*this, readFile(recordFileName), top); + Value functions; + getAttr(top, functionsSymbol, functions); + forceList(functions); + for (unsigned int i = 0; i < functions.listSize(); ++i) { + Value ¤t = *functions.listElems()[i]; + Value name, parameters, result; + getAttr(current, nameSymbol, name); + getAttr(current, parametersSymbol, parameters); + getAttr(current, resultSymbol, result); + std::string nameString = forceStringNoCtx(name); + std::list parameterList; + forceList(parameters); + for (unsigned int j = 0; j < parameters.listSize(); ++j) { + parameterList.push_back(valueToJSON(*parameters.listElems()[j])); + } + recording[std::make_pair(nameString, parameterList)] = result; + } + Value sources; + getAttr(top, sourcesSymbol, sources); + forceAttrs(sources); + std::cout << sources.attrs->size() << std::endl; + for (auto it = sources.attrs->begin(); it != sources.attrs->end(); ++it) { + std::cout << it->name << " -> " << forceStringNoCtx(*it->value) << std::endl; + srcToStore[it->name] = forceStringNoCtx(*it->value); + } +} EvalState::~EvalState() { fileEvalCache.clear(); if (evalMode == Record) { - std::fstream out(recordFileName, std::fstream::out); - out << "{\"functions\": [\n"; - bool isThisTheFirstTime = true; - - for (auto kv : recording) { - std::cout << "?"; - if (!isThisTheFirstTime) out << ","; - isThisTheFirstTime = false; - - out << "{ \"name\": \"" << kv.first.first << "\", \"parameters\": ["; - - bool isThisTheFirstTime2 = true; - for (auto parameter: kv.first.second) { - if (!isThisTheFirstTime2) out << ", "; - isThisTheFirstTime2 = false; - out << parameter; - } - out << "], \"result\": " << valueToJSON(kv.second) << "}\n"; - } - - out << "], \"sources\": {"; - - bool isThisTheFirstTime3 = true; - for (auto path : srcToStore) { - if (!isThisTheFirstTime3) out << ", "; - isThisTheFirstTime3 = false; - out << "\"" << path.first << "\": \"" << path.second << "\""; - } - out << "}}"; - out.close(); + finalizeRecord(); + } +} + +void EvalState::finalizeRecord() +{ + std::ofstream out(recordFileName, std::fstream::out); + out << "{\"functions\": [\n"; + + //TODO: write this in a more functional style + bool isThisTheFirstTime = true; + for (auto kv : recording) { + if (!isThisTheFirstTime) out << ","; + isThisTheFirstTime = false; + + out << "{ \"name\": \"" << kv.first.first << "\", \"parameters\": ["; + + bool isThisTheFirstTime2 = true; + for (auto parameter: kv.first.second) { + if (!isThisTheFirstTime2) out << ", "; + isThisTheFirstTime2 = false; + out << parameter; + } + out << "], \"result\": " << valueToJSON(kv.second) << "}\n"; + } + + out << "], \"sources\": {"; + + bool isThisTheFirstTime3 = true; + for (auto path : srcToStore) { + if (!isThisTheFirstTime3) out << ", "; + isThisTheFirstTime3 = false; + out << "\"" << path.first << "\": \"" << path.second << "\""; } + out << "}}"; + out.close(); } - Path EvalState::checkSourcePath(const Path & path_) { if (!restricted) return path_; @@ -439,7 +452,6 @@ std::string EvalState::valueToJSON(Value & value) { return out.str(); } - void EvalState::getBuiltin(const string & name, Value & v) { v = *baseEnv.values[0]->attrs->find(symbols.create(name))->value; @@ -716,13 +728,11 @@ void EvalState::resetFileCache() } -void EvalState::getAttr(Value & attrSet, const char * attr, Value & v) { +void EvalState::getAttr(Value & attrSet, const Symbol & attr, Value & v) { forceAttrs(attrSet); - // !!! Should we create a symbol here or just do a lookup? - Bindings::iterator i = attrSet.attrs->find(symbols.create(attr)); + Bindings::iterator i = attrSet.attrs->find(attr); if (i == attrSet.attrs->end()) throwEvalError("attribute ‘%1%’ missing", attr); - // !!! add to stack trace? forceValue(*i->value); v = *i->value; } @@ -1549,9 +1559,9 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path, bool ign if (srcToStore[path] != "") dstPath = srcToStore[path]; else { - if (evalMode == Playback) { - throwEvalError("Unknown path encountered in playback mode: '%1%'", path); - } + if (evalMode == Playback) { + throwEvalError("Unknown path encountered in playback mode: '%1%'", path); + } dstPath = (settings.readOnlyMode && !ignoreReadOnly) ? computeStorePathForPath(checkSourcePath(path)).first : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0cedf4ed791..0643ed1ca5b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -5,7 +5,6 @@ #include "nixexpr.hh" #include "symbol-table.hh" #include "hash.hh" -#include #include @@ -23,7 +22,7 @@ typedef enum { Normal, Record, Playback -} EvalMode; +} DeterministicEvaluationMode; typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); @@ -101,10 +100,11 @@ private: SearchPath searchPath; - EvalMode evalMode; + DeterministicEvaluationMode evalMode; const char * recordFileName; //TODO: use some other structure, maybe a hashmap + //since comparing strings with long same prefixes is slow std::map>, Value> recording; public: @@ -205,52 +205,69 @@ private: unsigned int arity, PrimOpFun primOp); std::string valueToJSON(Value & value); - - template< PrimOpFun primOp, const char *name, unsigned int arity> - static void recordPrimOp(EvalState & state, const Pos & pos, Value * * args, Value & v) - { - std::list argList; - for (int i = 0; i < arity; i++) { - argList.push_back(state.valueToJSON(*args[i])); - } - std::cout << "Record " << name << std::endl; - primOp(state, pos, args, v); - state.recording[std::make_pair(name, argList)] = v; - } - - template< PrimOpFun primOp, const char *name, unsigned int arity> + void getAttr(Value & top, const Symbol & arg2, Value & v); + void initializeDeterministicEvaluationMode(); + void initializePlayback(); + void finalizeRecord(); + + template< const char * name, unsigned int arity, PrimOpFun primOp> + static void recordPrimOp(EvalState & state, const Pos & pos, Value * * args, Value & v) + { + std::list argList; + for (int i = 0; i < arity; i++) { + argList.push_back(state.valueToJSON(*args[i])); + } + primOp(state, pos, args, v); + state.recording[std::make_pair(name, argList)] = v; + } + + template< const char * name, unsigned int arity, PrimOpFun primOp> static void playbackPrimOp(EvalState & state, const Pos & pos, Value * * args, Value & v) - { - std::list argList; - for (int i = 0; i < arity; i++) { - argList.push_back(state.valueToJSON(*args[i])); - } - std::cout << "Playback " << name << std::endl; - auto result = state.recording.find(std::make_pair(name, argList)); - if (result == state.recording.end()) { - std::string errorMsg("wanted to call "); - errorMsg +=name; - throw EvalError(errorMsg.c_str()); - } - else { - v = result->second; - } + { + std::list argList; + for (int i = 0; i < arity; i++) { + argList.push_back(state.valueToJSON(*args[i])); + } + auto result = state.recording.find(std::make_pair(name, argList)); + if (result == state.recording.end()) { + std::string errorMsg("wanted to call "); + errorMsg +=name; + throw EvalError(errorMsg.c_str()); + } + else { + v = result->second; + } } - - template< const char *name, unsigned int arity, PrimOpFun primOp> + + template< const char * name, unsigned int arity, PrimOpFun primOp> void addImpurePrimOp() - { - if (evalMode == Record) { - PrimOpFun wrapped = recordPrimOp; - addPrimOp(name, arity, wrapped); - } else if (evalMode == Playback) { - PrimOpFun wrapped = playbackPrimOp; - addPrimOp(name, arity, wrapped); - } else { - addPrimOp(name, arity, primOp); - } + { + if (evalMode == Record) { + addPrimOp(name, arity, recordPrimOp); + } else if (evalMode == Playback) { + addPrimOp(name, arity, playbackPrimOp); + } else { + addPrimOp(name, arity, primOp); + } } + template< const char * name > + static void unsupportedPrimOp(EvalState & state, const Pos & pos, Value * * args, Value & v) + { + throw EvalError(format("primop '%s' is not (yet) supported in Record/Playback mode (used at '%s')") % name % pos); + } + + template< const char * name > + void addUnsupportedImpurePrimOp(unsigned int arity, PrimOpFun primOp) + { + if (evalMode == Record || evalMode == Playback) { + addPrimOp(name, arity, unsupportedPrimOp); + } + else { + addPrimOp(name, arity, primOp); + } + } + public: void getBuiltin(const string & name, Value & v); @@ -330,7 +347,6 @@ private: friend struct ExprOpConcatLists; friend struct ExprSelect; friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); - void getAttr(Value & top, const char* arg2, Value & v); }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 7813fe943d5..59d9f8f1f10 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -576,10 +576,10 @@ Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv) { Path actualPath; if (evalMode == Record || evalMode == Playback) { - PathSet context; - actualPath = copyPathToStore(context, path, true); + PathSet context; + actualPath = copyPathToStore(context, path, true); } else { - actualPath = path; + actualPath = path; } return parse(readFile(actualPath).c_str(), path, dirOf(path), staticEnv); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 72ca15aff17..52ecba03692 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1706,6 +1706,7 @@ extern const char __getEnv[] = "__getEnv"; extern const char __storePath[] = "__storePath"; extern const char __pathExists[] = "__pathExists"; extern const char __readFile[] = "__readFile"; +extern const char __filterSource[] = "__filterSource"; void EvalState::createBaseEnv() @@ -1792,9 +1793,7 @@ void EvalState::createBaseEnv() addPrimOp("__toJSON", 1, prim_toJSON); addPrimOp("__fromJSON", 1, prim_fromJSON); addPrimOp("__toFile", 2, prim_toFile); - - //TODO: filter source is an impure primop too - addPrimOp("__filterSource", 2, prim_filterSource); + addUnsupportedImpurePrimOp<__filterSource>(2, prim_filterSource); // Sets addPrimOp("__attrNames", 1, prim_attrNames); @@ -1851,8 +1850,8 @@ void EvalState::createBaseEnv() addPrimOp("derivationStrict", 1, prim_derivationStrict); // Networking - addPrimOp("__fetchurl", 1, prim_fetchurl); - addPrimOp("fetchTarball", 1, prim_fetchTarball); + addImpurePrimOp("__fetchurl", 1, prim_fetchurl); + addImpurePrimOp("fetchTarball", 1, prim_fetchTarball); /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ From b89506312844d2d879598095814090f401465eb0 Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Thu, 19 Nov 2015 01:46:40 +0000 Subject: [PATCH 08/20] first version of working constants (currently fails when constants change because the result included in the primop argument) --- src/libexpr/eval.cc | 56 ++++++++++++++++++++++++++++++++++++------ src/libexpr/eval.hh | 31 ++++++++++++++--------- src/libexpr/primops.cc | 18 ++++++++------ 3 files changed, 79 insertions(+), 26 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1ea32566d8a..585fab7edc5 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -419,17 +419,60 @@ Path EvalState::checkSourcePath(const Path & path_) throw RestrictedPathError(format("access to path ‘%1%’ is forbidden in restricted mode") % path_); } +void EvalState::addToBaseEnv(const string & name, Value * v, Symbol sym) +{ + staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; + baseEnv.values[baseEnvDispl++] = v; + baseEnv.values[0]->attrs->push_back(Attr(sym, v)); +} + +void EvalState::addToBaseEnv(const string & name, Value * v) +{ + string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + addToBaseEnv(name, v, symbols.create(name2)); +} void EvalState::addConstant(const string & name, Value & v) { Value * v2 = allocValue(); *v2 = v; - staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; - baseEnv.values[baseEnvDispl++] = v2; - string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; - baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2)); + addToBaseEnv(name, v2); +} + +void EvalState::addImpureConstant(const string & name, Value & v, Value * constantPrimOp) +{ + if (evalMode == Normal) { + addConstant(name, v); + return; + } + Value * v2 = allocValue(); + *v2 = v; + Value * wrappedConstant = allocValue(); + wrappedConstant->type = tApp; + Value * left = wrappedConstant->app.left = allocValue(); + left->type = tPrimOpApp; + left->primOpApp.left = constantPrimOp; + left->primOpApp.right = allocValue(); + mkString(*left->primOpApp.right, name); + wrappedConstant->primOpApp.right = v2; + forceValue(*wrappedConstant); + addToBaseEnv(name, wrappedConstant); } +static void prim_impureConstant(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + v = *args[1]; +} + +extern const char __impureConstant[] = "__impureConstant"; +Value * EvalState::getImpureConstantPrimop() +{ + if (evalMode == Normal) return 0; + Value * result = allocValue(); + result->type = tPrimOp; + result->primOp = NEW PrimOp(transformPrimOp<__impureConstant, 2, prim_impureConstant>(), 2, symbols.create("impureConstant")); + return result; +} void EvalState::addPrimOp(const string & name, unsigned int arity, PrimOpFun primOp) @@ -439,12 +482,9 @@ void EvalState::addPrimOp(const string & name, Symbol sym = symbols.create(name2); v->type = tPrimOp; v->primOp = NEW PrimOp(primOp, arity, sym); - staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; - baseEnv.values[baseEnvDispl++] = v; - baseEnv.values[0]->attrs->push_back(Attr(sym, v)); + addToBaseEnv(name, v, sym); } - std::string EvalState::valueToJSON(Value & value) { std::ostringstream out; PathSet context; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0643ed1ca5b..e9064c16f0c 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -200,7 +200,8 @@ private: void createBaseEnv(); void addConstant(const string & name, Value & v); - + void addImpureConstant(const string & name, Value & v, Value * impureConstant); + void addPrimOp(const string & name, unsigned int arity, PrimOpFun primOp); @@ -214,7 +215,7 @@ private: static void recordPrimOp(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::list argList; - for (int i = 0; i < arity; i++) { + for (unsigned int i = 0; i < arity; i++) { argList.push_back(state.valueToJSON(*args[i])); } primOp(state, pos, args, v); @@ -225,7 +226,7 @@ private: static void playbackPrimOp(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::list argList; - for (int i = 0; i < arity; i++) { + for (unsigned int i = 0; i < arity; i++) { argList.push_back(state.valueToJSON(*args[i])); } auto result = state.recording.find(std::make_pair(name, argList)); @@ -238,17 +239,22 @@ private: v = result->second; } } - + + template< const char * name, unsigned int arity, PrimOpFun primOp> + PrimOpFun transformPrimOp() + { + switch (evalMode) { + case Record: return recordPrimOp; + case Playback: return playbackPrimOp; + case Normal: return primOp; + default: abort(); + } + } + template< const char * name, unsigned int arity, PrimOpFun primOp> void addImpurePrimOp() { - if (evalMode == Record) { - addPrimOp(name, arity, recordPrimOp); - } else if (evalMode == Playback) { - addPrimOp(name, arity, playbackPrimOp); - } else { - addPrimOp(name, arity, primOp); - } + addPrimOp(name, arity, transformPrimOp()); } template< const char * name > @@ -347,6 +353,9 @@ private: friend struct ExprOpConcatLists; friend struct ExprSelect; friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); + void addToBaseEnv(const string & name, Value * v, Symbol sym); + void addToBaseEnv(const string & name, Value * v); + Value * getImpureConstantPrimop(); }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 52ecba03692..e6adaa5d8ad 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1699,6 +1699,7 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args * Primop registration *************************************************************/ + extern const char __importNative[] = "__importNative"; extern const char __findFile[] = "__findFile"; extern const char __readDir[] = "__readDir"; @@ -1707,7 +1708,8 @@ extern const char __storePath[] = "__storePath"; extern const char __pathExists[] = "__pathExists"; extern const char __readFile[] = "__readFile"; extern const char __filterSource[] = "__filterSource"; - +extern const char __fetchurl[] = "__fetchurl"; +extern const char fetchTarball[] = "fetchTarball"; void EvalState::createBaseEnv() { @@ -1728,18 +1730,20 @@ void EvalState::createBaseEnv() mkNull(v); addConstant("null", v); + + Value * impureConstantPrimop = getImpureConstantPrimop(); mkInt(v, time(0)); - addConstant("__currentTime", v); + addImpureConstant("__currentTime", v, impureConstantPrimop); mkString(v, settings.thisSystem); - addConstant("__currentSystem", v); + addImpureConstant("__currentSystem", v, impureConstantPrimop); mkString(v, nixVersion); - addConstant("__nixVersion", v); + addImpureConstant("__nixVersion", v, impureConstantPrimop); mkString(v, settings.nixStore); - addConstant("__storeDir", v); + addImpureConstant("__storeDir", v, impureConstantPrimop); /* Language version. This should be increased every time a new language feature gets added. It's not necessary to increase it @@ -1850,8 +1854,8 @@ void EvalState::createBaseEnv() addPrimOp("derivationStrict", 1, prim_derivationStrict); // Networking - addImpurePrimOp("__fetchurl", 1, prim_fetchurl); - addImpurePrimOp("fetchTarball", 1, prim_fetchTarball); + addImpurePrimOp<__fetchurl, 1, prim_fetchurl>(); + addImpurePrimOp(); /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ From cf8b63be333dfa6743c2c2d3f62ab325bd5872fa Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Thu, 19 Nov 2015 01:58:14 +0000 Subject: [PATCH 09/20] fix last commit (don't force the constant, otherwise all will be recorded...) --- src/libexpr/eval.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 585fab7edc5..dad746e9f0b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -455,7 +455,6 @@ void EvalState::addImpureConstant(const string & name, Value & v, Value * consta left->primOpApp.right = allocValue(); mkString(*left->primOpApp.right, name); wrappedConstant->primOpApp.right = v2; - forceValue(*wrappedConstant); addToBaseEnv(name, wrappedConstant); } From b14a68c9ce012bf6a2ee241b58120987f12e41d7 Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Thu, 19 Nov 2015 06:01:17 +0000 Subject: [PATCH 10/20] fix bug that toJSON/fromJSON wasn't inverse because the content got copied to the store. Well, they're still not inverse because a path gets converted to a string, but that doesn't seem to be a problem so far --- src/libexpr/eval.cc | 10 ++++------ src/libexpr/eval.hh | 6 +++--- src/libexpr/value-to-json.cc | 10 +++++++--- src/libexpr/value-to-json.hh | 2 +- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index dad746e9f0b..0ed8394a8f1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -336,16 +336,14 @@ void EvalState::initializePlayback() std::list parameterList; forceList(parameters); for (unsigned int j = 0; j < parameters.listSize(); ++j) { - parameterList.push_back(valueToJSON(*parameters.listElems()[j])); + parameterList.push_back(valueToJSON(*parameters.listElems()[j], false)); } recording[std::make_pair(nameString, parameterList)] = result; } Value sources; getAttr(top, sourcesSymbol, sources); forceAttrs(sources); - std::cout << sources.attrs->size() << std::endl; for (auto it = sources.attrs->begin(); it != sources.attrs->end(); ++it) { - std::cout << it->name << " -> " << forceStringNoCtx(*it->value) << std::endl; srcToStore[it->name] = forceStringNoCtx(*it->value); } } @@ -378,7 +376,7 @@ void EvalState::finalizeRecord() isThisTheFirstTime2 = false; out << parameter; } - out << "], \"result\": " << valueToJSON(kv.second) << "}\n"; + out << "], \"result\": " << valueToJSON(kv.second, false) << "}\n"; } out << "], \"sources\": {"; @@ -484,10 +482,10 @@ void EvalState::addPrimOp(const string & name, addToBaseEnv(name, v, sym); } -std::string EvalState::valueToJSON(Value & value) { +std::string EvalState::valueToJSON(Value & value, bool copyToStore) { std::ostringstream out; PathSet context; - printValueAsJSON(*this, true, value, out, context); + printValueAsJSON(*this, true, value, out, context, copyToStore); return out.str(); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e9064c16f0c..3858b73abc9 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -205,7 +205,7 @@ private: void addPrimOp(const string & name, unsigned int arity, PrimOpFun primOp); - std::string valueToJSON(Value & value); + std::string valueToJSON(Value & value, bool copyToStore); void getAttr(Value & top, const Symbol & arg2, Value & v); void initializeDeterministicEvaluationMode(); void initializePlayback(); @@ -216,7 +216,7 @@ private: { std::list argList; for (unsigned int i = 0; i < arity; i++) { - argList.push_back(state.valueToJSON(*args[i])); + argList.push_back(state.valueToJSON(*args[i], false)); } primOp(state, pos, args, v); state.recording[std::make_pair(name, argList)] = v; @@ -227,7 +227,7 @@ private: { std::list argList; for (unsigned int i = 0; i < arity; i++) { - argList.push_back(state.valueToJSON(*args[i])); + argList.push_back(state.valueToJSON(*args[i], false)); } auto result = state.recording.find(std::make_pair(name, argList)); if (result == state.recording.end()) { diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index b0cf85e21f1..267bf2c1c71 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -25,7 +25,7 @@ void escapeJSON(std::ostream & str, const string & s) void printValueAsJSON(EvalState & state, bool strict, - Value & v, std::ostream & str, PathSet & context) + Value & v, std::ostream & str, PathSet & context, bool copyPathToStore) { checkInterrupt(); @@ -47,9 +47,13 @@ void printValueAsJSON(EvalState & state, bool strict, break; case tPath: - escapeJSON(str, state.copyPathToStore(context, v.path)); + if(copyPathToStore) { + escapeJSON(str, state.copyPathToStore(context, v.path)); + } + else { + escapeJSON(str, v.path); + } break; - case tNull: str << "null"; break; diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index f6796f2053e..b4adb3912f2 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -9,7 +9,7 @@ namespace nix { void printValueAsJSON(EvalState & state, bool strict, - Value & v, std::ostream & out, PathSet & context); + Value & v, std::ostream & out, PathSet & context, bool copyPathToStore = true); void escapeJSON(std::ostream & str, const string & s); From 46520daa7e59aaaa2bb6c5bf5c649e296a35587a Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Thu, 19 Nov 2015 12:38:56 +0000 Subject: [PATCH 11/20] also write information into the json file, so the command can be replayed currently only supported for nix-instantiate (and thus probably also nix-build), not yet nix-env add --playback option to play back the exact command So the interface is now: "NIX_RECORDING=filename nix-instantiate ..." to record the nix-instantiate command and "nix-instantiate --replay filename" to play it back commands in file --- src/libexpr/common-opts.cc | 4 +- src/libexpr/common-opts.hh | 2 +- src/libexpr/eval.cc | 120 +++++++++++++++++++++---- src/libexpr/eval.hh | 16 +++- src/nix-instantiate/nix-instantiate.cc | 37 +++++--- 5 files changed, 142 insertions(+), 37 deletions(-) diff --git a/src/libexpr/common-opts.cc b/src/libexpr/common-opts.cc index 13760490d9c..c4f5e5ed288 100644 --- a/src/libexpr/common-opts.cc +++ b/src/libexpr/common-opts.cc @@ -52,7 +52,7 @@ bool parseSearchPathArg(Strings::iterator & i, } -Path lookupFileArg(EvalState & state, string s) +Path lookupFileArg(EvalState & state, string s, string currentPath) { if (isUri(s)) return downloadFileCached(s, true); @@ -60,7 +60,7 @@ Path lookupFileArg(EvalState & state, string s) Path p = s.substr(1, s.size() - 2); return state.findFile(p); } else - return absPath(s); + return absPath(s, currentPath); } diff --git a/src/libexpr/common-opts.hh b/src/libexpr/common-opts.hh index be0f4020243..248eeeee8a9 100644 --- a/src/libexpr/common-opts.hh +++ b/src/libexpr/common-opts.hh @@ -13,6 +13,6 @@ Bindings * evalAutoArgs(EvalState & state, std::map & in); bool parseSearchPathArg(Strings::iterator & i, const Strings::iterator & argsEnd, Strings & searchPath); -Path lookupFileArg(EvalState & state, string s); +Path lookupFileArg(EvalState & state, string s, string currentPath = ""); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0ed8394a8f1..d8f692915b1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -7,6 +7,7 @@ #include "eval-inline.hh" #include "value-to-json.hh" #include "json-to-value.hh" +#include "common-opts.hh" #include #include @@ -249,7 +250,7 @@ static Strings parseNixPath(const string & in) } -EvalState::EvalState(const Strings & _searchPath) +EvalState::EvalState(const Strings & _searchPath, DeterministicEvaluationMode evalMode, const char * evalModeFile) : sWith(symbols.create("")) , sOutPath(symbols.create("outPath")) , sDrvPath(symbols.create("drvPath")) @@ -267,7 +268,8 @@ EvalState::EvalState(const Strings & _searchPath) , sColumn(symbols.create("column")) , sFunctor(symbols.create("__functor")) , sToString(symbols.create("__toString")) - , evalMode(Normal) + , evalMode(evalMode) + , recordFileName(evalModeFile) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) { @@ -293,26 +295,86 @@ EvalState::EvalState(const Strings & _searchPath) void EvalState::initializeDeterministicEvaluationMode() { - const char * recordMode = getenv("NIX_RECORDING"); - const char * playbackMode = getenv("NIX_PLAYBACK"); - if (recordMode && !playbackMode) { - evalMode = Record; - recordFileName = recordMode; - } else if (playbackMode && !recordMode) { - evalMode = Playback; - recordFileName = playbackMode; + if (evalMode == Normal) { + const char * recordMode = getenv("NIX_RECORDING"); + const char * playbackMode = getenv("NIX_PLAYBACK"); + if (recordMode && !playbackMode) { + evalMode = Record; + recordFileName = recordMode; + } else if (playbackMode && !recordMode) { + evalMode = Playback; + recordFileName = playbackMode; + } else if (!playbackMode && !recordMode) { + evalMode = Normal; + } + else + throw EvalError("can't use both NIX_RECORDING and NIX_PLAYBACK"); + } + if (evalMode == Playback || evalMode == Record) + assert(recordFileName != 0); + + if (evalMode == Playback) { initializePlayback(); - } else if (!playbackMode && !recordMode) { - evalMode = Normal; } - else - throw EvalError("can't use both NIX_RECORDING and NIX_PLAYBACK"); if (evalMode == Record || evalMode == Playback) { std::cerr << "Running in deterministic evaluation mode: " << evalMode << std::endl; } } +void EvalState::setRecordingInfo(bool fromArgs, std::map autoArgs_, Strings attrPaths, Strings files, string currentDir) +{ + recordingExpression.fromArgs = fromArgs; + recordingExpression.autoArgs = autoArgs_; + recordingExpression.attrPaths = attrPaths; + recordingExpression.files = files; + recordingExpression.currentDir = currentDir; +} + +void EvalState::getRecordingInfo(bool & fromArgs, std::map & autoArgs_, Strings & attrPaths, Strings & files, string & currentDir) +{ + if (!autoArgs_.empty()) { + throw Error("you can't supply auto arguments in playback mode"); + } + if (!attrPaths.empty()) { + throw Error("you can't supply attribute paths in playback mode"); + } + if (!files.empty()) { + throw Error("you can't supply files or expressions in playback mode"); + } + if (fromArgs) { + throw Error("--expr is invalid in playback mode"); + } + Symbol expressionS(symbols.create("expression")), + fromArgsS(symbols.create("fromArgs")), + autoArgsS(symbols.create("autoArgs")), + attributesS(symbols.create("attributes")), + filesS(symbols.create("files")), + currentDirS(symbols.create("currentDir")); + Value expression, fromArgsV, autoArgsV, attributesV, filesV, currentDirV; + getAttr(*playbackJson, expressionS, expression); + getAttr(expression, fromArgsS, fromArgsV); + getAttr(expression, autoArgsS, autoArgsV); + getAttr(expression, attributesS, attributesV); + getAttr(expression, filesS, filesV); + getAttr(expression, currentDirS, currentDirV); + fromArgs = forceBool(fromArgsV); + forceAttrs(autoArgsV); + for (auto it = autoArgsV.attrs->begin(); it != autoArgsV.attrs->end(); ++it) { + forceString(*it->value); + autoArgs_[it->name] = forceStringNoCtx(*it->value); + } + forceList(attributesV); + for (unsigned int i = 0; i < attributesV.listSize(); ++i) { + attrPaths.push_back(forceStringNoCtx(*attributesV.listElems()[i])); + } + forceList(filesV); + for (unsigned int i = 0; i < filesV.listSize(); ++i) { + files.push_back(forceStringNoCtx(*filesV.listElems()[i])); + } + currentDir = forceStringNoCtx(currentDirV); +} + void EvalState::initializePlayback() { Symbol functionsSymbol(symbols.create("functions")), @@ -320,7 +382,8 @@ void EvalState::initializePlayback() parametersSymbol(symbols.create("parameters")), resultSymbol(symbols.create("result")), sourcesSymbol(symbols.create("sources")); - Value top; + Value & top (*allocValue()); + playbackJson = ⊤ parseJSON(*this, readFile(recordFileName), top); Value functions; @@ -359,10 +422,33 @@ EvalState::~EvalState() void EvalState::finalizeRecord() { + //TODO: write this in a more functional style std::ofstream out(recordFileName, std::fstream::out); - out << "{\"functions\": [\n"; + out << "{\"expression\":{\"fromArgs\":"; + out << (recordingExpression.fromArgs ? "true" : "false"); + out << ",\"autoArgs\":{"; + bool isThisTheFirstTime0 = true; + for (auto autoArg: recordingExpression.autoArgs) { + if (!isThisTheFirstTime0) out << ","; + isThisTheFirstTime0 = false; + out << "\"" << autoArg.first << "\":\"" << autoArg.second << "\""; + } + out << "},\"attributes\":["; + isThisTheFirstTime0 = true; + for (auto attr: recordingExpression.attrPaths) { + if (!isThisTheFirstTime0) out << ","; + isThisTheFirstTime0 = false; + out << "\"" << attr << "\""; + } + out << "],\"files\":["; + isThisTheFirstTime0 = true; + for (auto file: recordingExpression.files) { + if (!isThisTheFirstTime0) out << ","; + isThisTheFirstTime0 = false; + out << "\"" << resolveExprPath(lookupFileArg(*this, file)) << "\""; + } + out << "],\"currentDir\":\"" << recordingExpression.currentDir << "\"},\"functions\": [\n"; - //TODO: write this in a more functional style bool isThisTheFirstTime = true; for (auto kv : recording) { if (!isThisTheFirstTime) out << ","; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 3858b73abc9..db9432313a7 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -26,7 +26,6 @@ typedef enum { typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); - struct PrimOp { PrimOpFun fun; @@ -102,14 +101,23 @@ private: DeterministicEvaluationMode evalMode; const char * recordFileName; + Value * playbackJson; //TODO: use some other structure, maybe a hashmap //since comparing strings with long same prefixes is slow std::map>, Value> recording; - + struct RecordingExpression { + bool fromArgs; + std::map autoArgs; + Strings attrPaths; + Strings files; + string currentDir; + RecordingExpression() : fromArgs(false) {} + } recordingExpression; + public: - EvalState(const Strings & _searchPath); + EvalState(const Strings & _searchPath, DeterministicEvaluationMode evalMode = Normal, const char * evalModeFile = nullptr); ~EvalState(); void addToSearchPath(const string & s, bool warn = false); @@ -275,6 +283,8 @@ private: } public: + void getRecordingInfo(bool & fromArgs, std::map & autoArgs_, Strings & attrPaths, Strings & files, string & currentDir); + void setRecordingInfo(bool fromArgs, std::map autoArgs_, Strings attrPaths, Strings files, string currentDir); void getBuiltin(const string & name, Value & v); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 13a145a3b53..c0784a3b16f 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -101,6 +101,7 @@ int main(int argc, char * * argv) bool findFile = false; bool evalOnly = false; bool parseOnly = false; + Path replay; OutputKind outputKind = okPlain; bool xmlOutputSourceLocation = true; bool strict = false; @@ -148,6 +149,8 @@ int main(int argc, char * * argv) repair = true; else if (*arg == "--dry-run") settings.readOnlyMode = true; + else if (*arg == "--replay") + replay = getArg(*arg, arg, end); else if (*arg != "" && arg->at(0) == '-') return false; else @@ -160,32 +163,38 @@ int main(int argc, char * * argv) store = openStore(); - EvalState state(searchPath); + EvalState state(searchPath, replay.empty() ? Normal : Playback, replay.c_str()); state.repair = repair; - - Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); - - if (attrPaths.empty()) attrPaths.push_back(""); - - if (findFile) { - for (auto & i : files) { - Path p = state.findFile(i); - if (p == "") throw Error(format("unable to find ‘%1%’") % i); - std::cout << p << std::endl; + string currentPath = absPath("."); + + if (replay.empty()) { + if (attrPaths.empty()) attrPaths.push_back(""); + + if (findFile) { + for (auto & i : files) { + Path p = state.findFile(i); + if (p == "") throw Error(format("unable to find ‘%1%’") % i); + std::cout << p << std::endl; + } + return; } - return; + + state.setRecordingInfo(fromArgs, autoArgs_, attrPaths, files, currentPath); + } + else { + state.getRecordingInfo(fromArgs, autoArgs_, attrPaths, files, currentPath); } + Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); if (readStdin) { Expr * e = parseStdin(state); processExpr(state, attrPaths, parseOnly, strict, autoArgs, evalOnly, outputKind, xmlOutputSourceLocation, e); } else if (files.empty() && !fromArgs) files.push_back("./default.nix"); - for (auto & i : files) { Expr * e = fromArgs - ? state.parseExprFromString(i, absPath(".")) + ? state.parseExprFromString(i, currentPath) : state.parseExprFromFile(resolveExprPath(lookupFileArg(state, i))); processExpr(state, attrPaths, parseOnly, strict, autoArgs, evalOnly, outputKind, xmlOutputSourceLocation, e); From 9c7dfec6fd699bf4ea4a8b4ac259a42fb1f3ffb3 Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Thu, 19 Nov 2015 23:59:18 +0000 Subject: [PATCH 12/20] save generated json file in the store with its proper dependencies --- corepkgs/local.mk | 2 +- src/libexpr/eval.cc | 35 ++++++++++++++++++++++++++++++----- src/libexpr/eval.hh | 3 ++- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/corepkgs/local.mk b/corepkgs/local.mk index 19c1d06962c..ccca3e15fba 100644 --- a/corepkgs/local.mk +++ b/corepkgs/local.mk @@ -1,4 +1,4 @@ -corepkgs_FILES = nar.nix buildenv.nix buildenv.pl unpack-channel.nix derivation.nix fetchurl.nix imported-drv-to-derivation.nix +corepkgs_FILES = nar.nix buildenv.nix buildenv.pl unpack-channel.nix derivation.nix fetchurl.nix imported-drv-to-derivation.nix reproducable-derivation.nix $(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs))) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d8f692915b1..6588750ef48 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -8,6 +8,7 @@ #include "value-to-json.hh" #include "json-to-value.hh" #include "common-opts.hh" +#include "get-drvs.hh" #include #include @@ -415,15 +416,38 @@ EvalState::~EvalState() { fileEvalCache.clear(); - if (evalMode == Record) { - finalizeRecord(); + if (evalMode == Record) { + Value result; + finalizeRecording(result); + writeRecordingIntoStore(result); } } -void EvalState::finalizeRecord() +void EvalState::writeRecordingIntoStore(Value & result) +{ + Value v; + string path = settings.nixDataDir + "/nix/corepkgs/reproducable-derivation.nix"; + evalFile(path, v); + Value app; + app.type = tApp; + app.app.left = &v; + app.app.right = &result; + DrvInfos drvs; + Bindings * autoArgs = allocBindings(4); + getDerivations(*this, app, "", *autoArgs, drvs, false); + Path drvPath = drvs.front().queryDrvPath(); + PathSet paths; + paths.insert(drvPath); + store->buildPaths(paths); + std::cerr << "succesfully build source closure: " << drvs.front().queryOutPath() << std::endl; +} + + +void EvalState::finalizeRecording(Value & result) { //TODO: write this in a more functional style - std::ofstream out(recordFileName, std::fstream::out); + std::stringstream out; + PathSet context; out << "{\"expression\":{\"fromArgs\":"; out << (recordingExpression.fromArgs ? "true" : "false"); out << ",\"autoArgs\":{"; @@ -471,10 +495,11 @@ void EvalState::finalizeRecord() for (auto path : srcToStore) { if (!isThisTheFirstTime3) out << ", "; isThisTheFirstTime3 = false; + context.insert(path.second); out << "\"" << path.first << "\": \"" << path.second << "\""; } out << "}}"; - out.close(); + mkString(result, out.str(), context); } Path EvalState::checkSourcePath(const Path & path_) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index db9432313a7..5800632c351 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -217,7 +217,8 @@ private: void getAttr(Value & top, const Symbol & arg2, Value & v); void initializeDeterministicEvaluationMode(); void initializePlayback(); - void finalizeRecord(); + void finalizeRecording (Value & result); + void writeRecordingIntoStore (Value & result); template< const char * name, unsigned int arity, PrimOpFun primOp> static void recordPrimOp(EvalState & state, const Pos & pos, Value * * args, Value & v) From a838853f5020e7021a6df7c2f49e369e18aa965b Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Fri, 20 Nov 2015 00:09:03 +0000 Subject: [PATCH 13/20] add --record command line parameter to nix-instantiate --- corepkgs/reproducable-derivation.nix | 14 ++++++++++++++ src/nix-instantiate/nix-instantiate.cc | 24 +++++++++++++++++------- 2 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 corepkgs/reproducable-derivation.nix diff --git a/corepkgs/reproducable-derivation.nix b/corepkgs/reproducable-derivation.nix new file mode 100644 index 00000000000..492c952b801 --- /dev/null +++ b/corepkgs/reproducable-derivation.nix @@ -0,0 +1,14 @@ +result: +with import ./config.nix; +derivation { + name = "source-closure"; + builder = shell; + args = ["-e" (__toFile "name" '' + #!/bin/bash + ${coreutils}/mkdir -p $out/nix-support + ${coreutils}/cat << EOF > $out/nix-support/source + ${result} + EOF + '')]; + system = builtins.currentSystem; +} \ No newline at end of file diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index c0784a3b16f..17587a75a1c 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -101,7 +101,8 @@ int main(int argc, char * * argv) bool findFile = false; bool evalOnly = false; bool parseOnly = false; - Path replay; + Path playback; + bool record = false; OutputKind outputKind = okPlain; bool xmlOutputSourceLocation = true; bool strict = false; @@ -149,8 +150,10 @@ int main(int argc, char * * argv) repair = true; else if (*arg == "--dry-run") settings.readOnlyMode = true; - else if (*arg == "--replay") - replay = getArg(*arg, arg, end); + else if (*arg == "--playback") + playback = getArg(*arg, arg, end); + else if (*arg == "--record") + record = true; else if (*arg != "" && arg->at(0) == '-') return false; else @@ -163,13 +166,21 @@ int main(int argc, char * * argv) store = openStore(); - EvalState state(searchPath, replay.empty() ? Normal : Playback, replay.c_str()); + DeterministicEvaluationMode mode = record ? Record : Normal; + if (!playback.empty()) { + if (mode == Normal) { + mode = Playback; + } + else + throw Error("can't specify both --playback and --record"); + } + + EvalState state(searchPath, mode, playback.c_str()); state.repair = repair; string currentPath = absPath("."); - if (replay.empty()) { + if (mode != Playback) { if (attrPaths.empty()) attrPaths.push_back(""); - if (findFile) { for (auto & i : files) { Path p = state.findFile(i); @@ -178,7 +189,6 @@ int main(int argc, char * * argv) } return; } - state.setRecordingInfo(fromArgs, autoArgs_, attrPaths, files, currentPath); } else { From 92d9e1b4fa624ea4ae7f6e4bfb072f003feb13ab Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Fri, 20 Nov 2015 01:31:10 +0000 Subject: [PATCH 14/20] add integration to nix-build/nix-shell fix json encoding --- scripts/nix-build.in | 16 +++++++++++++++- src/libexpr/eval.cc | 18 ++++++++++++------ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/scripts/nix-build.in b/scripts/nix-build.in index b93e5ab1390..30ef7db6163 100755 --- a/scripts/nix-build.in +++ b/scripts/nix-build.in @@ -18,6 +18,7 @@ my $pure = 0; my $fromArgs = 0; my $packages = 0; my $interactive = 1; +my $playback = 0; my @instArgs = (); my @buildArgs = (); @@ -124,6 +125,17 @@ for (my $n = 0; $n < scalar @ARGV; $n++) { $n += 2; } + elsif ($arg eq "--playback") { + die "$0: '$arg' requires an argument\n" unless $n + 1 < scalar @ARGV; + push @instArgs, ($arg, $ARGV[$n + 1]); + $playback = 1; + $n++; + } + + elsif ($arg eq "--record") { + push @instArgs, $arg; + } + elsif ($arg eq "--max-jobs" || $arg eq "-j" || $arg eq "--max-silent-time" || $arg eq "--log-type" || $arg eq "--cores" || $arg eq "--timeout" || $arg eq '--add-root') { $n++; die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; @@ -219,6 +231,7 @@ for (my $n = 0; $n < scalar @ARGV; $n++) { } die "$0: ‘-p’ and ‘-E’ are mutually exclusive\n" if $packages && $fromArgs; +die "$0: with --playback you can't supply expressions\n" if $playback && (scalar @exprs > 0 || $packages); if ($packages) { push @instArgs, "--expr"; @@ -243,8 +256,9 @@ foreach my $expr (@exprs) { $expr = dirname(Cwd::abs_path($script)) . "/" . $expr if $inShebang && !$packages && $expr !~ /^\//; + my @expression = $playback ? () : ($expr); # !!! would prefer the perl 5.8.0 pipe open feature here. - my $pid = open(DRVPATHS, "-|") || exec "$Nix::Config::binDir/nix-instantiate", "--add-root", $drvLink, "--indirect", @instArgs, $expr; + my $pid = open(DRVPATHS, "-|") || exec "$Nix::Config::binDir/nix-instantiate", "--add-root", $drvLink, "--indirect", @instArgs, @expression; while () {chomp; push @drvPaths, $_;} if (!close DRVPATHS) { die "nix-instantiate killed by signal " . ($? & 127) . "\n" if ($? & 127); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 6588750ef48..bf3dee9b423 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -455,21 +455,23 @@ void EvalState::finalizeRecording(Value & result) for (auto autoArg: recordingExpression.autoArgs) { if (!isThisTheFirstTime0) out << ","; isThisTheFirstTime0 = false; - out << "\"" << autoArg.first << "\":\"" << autoArg.second << "\""; + escapeJSON (out, autoArg.first); + out << ":"; + escapeJSON(out, autoArg.second); } out << "},\"attributes\":["; isThisTheFirstTime0 = true; for (auto attr: recordingExpression.attrPaths) { if (!isThisTheFirstTime0) out << ","; isThisTheFirstTime0 = false; - out << "\"" << attr << "\""; + escapeJSON(out, attr); } out << "],\"files\":["; isThisTheFirstTime0 = true; for (auto file: recordingExpression.files) { if (!isThisTheFirstTime0) out << ","; isThisTheFirstTime0 = false; - out << "\"" << resolveExprPath(lookupFileArg(*this, file)) << "\""; + escapeJSON(out, recordingExpression.fromArgs ? file : resolveExprPath(lookupFileArg(*this, file))); } out << "],\"currentDir\":\"" << recordingExpression.currentDir << "\"},\"functions\": [\n"; @@ -478,13 +480,15 @@ void EvalState::finalizeRecording(Value & result) if (!isThisTheFirstTime) out << ","; isThisTheFirstTime = false; - out << "{ \"name\": \"" << kv.first.first << "\", \"parameters\": ["; + out << "{ \"name\":"; + escapeJSON(out, kv.first.first); + out << ", \"parameters\": ["; bool isThisTheFirstTime2 = true; for (auto parameter: kv.first.second) { if (!isThisTheFirstTime2) out << ", "; isThisTheFirstTime2 = false; - out << parameter; + out << parameter; } out << "], \"result\": " << valueToJSON(kv.second, false) << "}\n"; } @@ -496,7 +500,9 @@ void EvalState::finalizeRecording(Value & result) if (!isThisTheFirstTime3) out << ", "; isThisTheFirstTime3 = false; context.insert(path.second); - out << "\"" << path.first << "\": \"" << path.second << "\""; + escapeJSON(out, path.first); + out << ":"; + escapeJSON(out, path.second); } out << "}}"; mkString(result, out.str(), context); From 1e69a65deb6544e4e73d1df5b811ab81c37de504 Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Fri, 20 Nov 2015 06:51:39 +0000 Subject: [PATCH 15/20] if import is in nix store, add package instead of individual files In combination with nix channels (or the newer http download), the whole channel is treated as one input --- src/libexpr/eval.cc | 44 ++++++++++++++++++++++++++++++++++++++------ src/libexpr/eval.hh | 4 +++- src/libexpr/parser.y | 2 +- 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index bf3dee9b423..e8d02f02129 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -408,7 +408,7 @@ void EvalState::initializePlayback() getAttr(top, sourcesSymbol, sources); forceAttrs(sources); for (auto it = sources.attrs->begin(); it != sources.attrs->end(); ++it) { - srcToStore[it->name] = forceStringNoCtx(*it->value); + srcToStoreForPlayback[it->name] = forceStringNoCtx(*it->value); } } @@ -479,7 +479,7 @@ void EvalState::finalizeRecording(Value & result) for (auto kv : recording) { if (!isThisTheFirstTime) out << ","; isThisTheFirstTime = false; - + out << "{ \"name\":"; escapeJSON(out, kv.first.first); out << ", \"parameters\": ["; @@ -496,7 +496,7 @@ void EvalState::finalizeRecording(Value & result) out << "], \"sources\": {"; bool isThisTheFirstTime3 = true; - for (auto path : srcToStore) { + for (auto path : srcToStoreForPlayback) { if (!isThisTheFirstTime3) out << ", "; isThisTheFirstTime3 = false; context.insert(path.second); @@ -1713,12 +1713,13 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path, bool ign if (srcToStore[path] != "") dstPath = srcToStore[path]; else { + Path path2 = path; if (evalMode == Playback) { - throwEvalError("Unknown path encountered in playback mode: '%1%'", path); + path2 = copyPathToStoreIfItsNotAlreadyThere(context, path); } dstPath = (settings.readOnlyMode && !ignoreReadOnly) - ? computeStorePathForPath(checkSourcePath(path)).first - : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); + ? computeStorePathForPath(checkSourcePath(path2)).first + : store->addToStore(baseNameOf(path), checkSourcePath(path2), true, htSHA256, defaultPathFilter, repair); srcToStore[path] = dstPath; printMsg(lvlChatty, format("copied source ‘%1%’ -> ‘%2%’") % path % dstPath); @@ -1728,6 +1729,37 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path, bool ign return dstPath; } +Path EvalState::copyPathToStoreIfItsNotAlreadyThere(PathSet context, Path path) +{ + if (evalMode == Playback) { + Path rest; + while (srcToStoreForPlayback[path] == "") { + string::size_type lastSeperator = path.find_last_of("/"); + if (lastSeperator == string::npos) { + throwEvalError("path '%s' not found in playback mode", path + rest); + } + rest = path.substr(lastSeperator) + rest; + path = path.substr(0, lastSeperator); + } + return srcToStoreForPlayback[path] + rest; + } + if (isInStore(path)) { + string storePath = toStorePath(path); + srcToStoreForPlayback[storePath] = storePath; + return path; + } + else { + if (srcToStoreForPlayback[path] != "") + return srcToStoreForPlayback[path]; + else { + string result = copyPathToStore(context, path, true); + srcToStoreForPlayback[path] = result; + return result; + } + } +} + + Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context) { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 5800632c351..181090af837 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -88,7 +88,8 @@ public: private: SrcToStore srcToStore; - + SrcToStore srcToStoreForPlayback; + /* A cache from path names to values. */ #if HAVE_BOEHMGC typedef std::map, traceable_allocator > > FileEvalCache; @@ -367,6 +368,7 @@ private: void addToBaseEnv(const string & name, Value * v, Symbol sym); void addToBaseEnv(const string & name, Value * v); Value * getImpureConstantPrimop(); + Path copyPathToStoreIfItsNotAlreadyThere(PathSet context, Path path); }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 59d9f8f1f10..b595e24e688 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -577,7 +577,7 @@ Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv) Path actualPath; if (evalMode == Record || evalMode == Playback) { PathSet context; - actualPath = copyPathToStore(context, path, true); + actualPath = copyPathToStoreIfItsNotAlreadyThere(context, path); } else { actualPath = path; } From 075869e138aaaccedd90fe329f8c98a9c6b864a9 Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Sat, 21 Nov 2015 22:51:25 +0000 Subject: [PATCH 16/20] only record relevant parameters to functions --- src/libexpr/eval.cc | 2 +- src/libexpr/eval.hh | 27 ++++++++++++++++++--------- src/libexpr/primops.cc | 2 +- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e8d02f02129..7a8b54440a2 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -584,7 +584,7 @@ Value * EvalState::getImpureConstantPrimop() if (evalMode == Normal) return 0; Value * result = allocValue(); result->type = tPrimOp; - result->primOp = NEW PrimOp(transformPrimOp<__impureConstant, 2, prim_impureConstant>(), 2, symbols.create("impureConstant")); + result->primOp = NEW PrimOp(transformPrimOp<__impureConstant, 2, prim_impureConstant, onlyPos<0> >(), 2, symbols.create("impureConstant")); return result; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 181090af837..04dd8f2c7aa 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -221,23 +221,32 @@ private: void finalizeRecording (Value & result); void writeRecordingIntoStore (Value & result); - template< const char * name, unsigned int arity, PrimOpFun primOp> + static bool constTrue(unsigned int arg) { return true; } + template< int argumentPos > + static bool onlyPos(unsigned int arg) { return arg == argumentPos; } + typedef bool (*UsedArguments) (unsigned int argumentIndex); + + template< const char * name, unsigned int arity, PrimOpFun primOp, UsedArguments useArgument > static void recordPrimOp(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::list argList; for (unsigned int i = 0; i < arity; i++) { - argList.push_back(state.valueToJSON(*args[i], false)); + if (useArgument(i)) { + argList.push_back(state.valueToJSON(*args[i], false)); + } } primOp(state, pos, args, v); state.recording[std::make_pair(name, argList)] = v; } - template< const char * name, unsigned int arity, PrimOpFun primOp> + template< const char * name, unsigned int arity, PrimOpFun primOp, UsedArguments useArgument > static void playbackPrimOp(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::list argList; for (unsigned int i = 0; i < arity; i++) { - argList.push_back(state.valueToJSON(*args[i], false)); + if(useArgument(i)) { + argList.push_back(state.valueToJSON(*args[i], false)); + } } auto result = state.recording.find(std::make_pair(name, argList)); if (result == state.recording.end()) { @@ -250,21 +259,21 @@ private: } } - template< const char * name, unsigned int arity, PrimOpFun primOp> + template< const char * name, unsigned int arity, PrimOpFun primOp, UsedArguments useArguments > PrimOpFun transformPrimOp() { switch (evalMode) { - case Record: return recordPrimOp; - case Playback: return playbackPrimOp; + case Record: return recordPrimOp; + case Playback: return playbackPrimOp; case Normal: return primOp; default: abort(); } } - template< const char * name, unsigned int arity, PrimOpFun primOp> + template< const char * name, unsigned int arity, PrimOpFun primOp, UsedArguments useArguments = constTrue> void addImpurePrimOp() { - addPrimOp(name, arity, transformPrimOp()); + addPrimOp(name, arity, transformPrimOp()); } template< const char * name > diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e6adaa5d8ad..29fa1a284a1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1790,7 +1790,7 @@ void EvalState::createBaseEnv() addPrimOp("dirOf", 1, prim_dirOf); addImpurePrimOp<__readFile, 1, prim_readFile>(); addImpurePrimOp<__readDir, 1, prim_readDir>(); - addImpurePrimOp<__findFile, 2, prim_findFile>(); + addImpurePrimOp<__findFile, 2, prim_findFile, onlyPos<1>>(); // Creating files addPrimOp("__toXML", 1, prim_toXML); From 955e81ef9faa35c2e5a8ed3e8a532549a3010b12 Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Sun, 22 Nov 2015 00:06:49 +0000 Subject: [PATCH 17/20] more robust escaping --- corepkgs/reproducable-derivation.nix | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/corepkgs/reproducable-derivation.nix b/corepkgs/reproducable-derivation.nix index 492c952b801..313a7f34565 100644 --- a/corepkgs/reproducable-derivation.nix +++ b/corepkgs/reproducable-derivation.nix @@ -6,9 +6,7 @@ derivation { args = ["-e" (__toFile "name" '' #!/bin/bash ${coreutils}/mkdir -p $out/nix-support - ${coreutils}/cat << EOF > $out/nix-support/source - ${result} - EOF + ${coreutils}/cp ${__toFile "result" result} $out/nix-support/source '')]; system = builtins.currentSystem; } \ No newline at end of file From eca9dec604c8fdfd27a4be7ca0b3b957d2ce1121 Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Fri, 22 Jan 2016 20:40:11 +0000 Subject: [PATCH 18/20] don't write a json, but write nix files we write 3 files to the nix closure: -> default.nix: just imports the other two and can be imported to get the exact reproducable output we want to have -> nix-support/recording.nix: recordings of all impurities i. e. function calls and imported files -> nix-support/expressions.nix: the expression that will be executed for the expression, a primop __findAlongAttrPath that does the supplies the autobindings along the attrpath. for the playback, a new primop prim_playback has been introduced. It's first parameter is a description of all impurities (as written to expression.nix). After this function is invoked, all subsequent calls can use the functions introduced here. In general, we allow multiple prim_playback when they don't conflict (not in all cases conflicts are found yet) It would also be cool to make the prim_playback more local, e. g. it should only apply to a specific term, but it's unclear to me how this can be achieved in a lazy language. --- corepkgs/reproducable-derivation.nix | 7 +- src/libexpr/eval.cc | 269 ++++++++++++++----------- src/libexpr/eval.hh | 45 +++-- src/libexpr/primops.cc | 26 +++ src/nix-instantiate/nix-instantiate.cc | 79 +++++--- 5 files changed, 258 insertions(+), 168 deletions(-) diff --git a/corepkgs/reproducable-derivation.nix b/corepkgs/reproducable-derivation.nix index 313a7f34565..0cb8c1d6478 100644 --- a/corepkgs/reproducable-derivation.nix +++ b/corepkgs/reproducable-derivation.nix @@ -1,4 +1,4 @@ -result: +{ recording, expressions}: with import ./config.nix; derivation { name = "source-closure"; @@ -6,7 +6,10 @@ derivation { args = ["-e" (__toFile "name" '' #!/bin/bash ${coreutils}/mkdir -p $out/nix-support - ${coreutils}/cp ${__toFile "result" result} $out/nix-support/source + ${coreutils}/cp ${__toFile "deterministic-recording" recording} $out/nix-support/recording.nix + ${coreutils}/cp ${__toFile "expressions" expressions} $out/nix-support/expressions.nix + echo "__playback { sources = { \"$out\" = ./.; }; functions = []; }" > $out/default.nix + echo "(__playback (import ./nix-support/recording.nix) (import ./nix-support/expressions.nix))" >> $out/default.nix '')]; system = builtins.currentSystem; } \ No newline at end of file diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7a8b54440a2..f006b5d0f56 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -142,6 +142,45 @@ static void printValue(std::ostream & str, std::set & active, con active.erase(&v); } +Expr * EvalState::valueToExpression(const Value & v) +{ + switch (v.type) { + case tInt: + return new ExprInt(v.integer); + case tBool: + return new ExprVar(symbols.create(v.boolean ? "true" : "false")); + case tString: + return new ExprString(symbols.create(v.string.s)); + case tPath: + return new ExprPath(v.path); + case tNull: + return new ExprVar(symbols.create("null")); + case tAttrs: + { + ExprAttrs * result = new ExprAttrs(); + for (auto a: *v.attrs) { + result->attrs.insert(std::make_pair(a.name, ExprAttrs::AttrDef(valueToExpression(*a.value), *a.pos))); + } + return result; + } + case tList1: + case tList2: + case tListN: + { + ExprList * result = new ExprList(); + for (int i = 0; i < v.listSize(); ++i) { + result->elems.push_back(valueToExpression(*v.listElems()[i])); + } + return result; + } + case tThunk: + return v.thunk.expr; + default: + throw Error("value not supported while converting"); + } +} + + std::ostream & operator << (std::ostream & str, const Value & v) { @@ -251,7 +290,7 @@ static Strings parseNixPath(const string & in) } -EvalState::EvalState(const Strings & _searchPath, DeterministicEvaluationMode evalMode, const char * evalModeFile) +EvalState::EvalState(const Strings & _searchPath, DeterministicEvaluationMode evalMode) : sWith(symbols.create("")) , sOutPath(symbols.create("outPath")) , sDrvPath(symbols.create("drvPath")) @@ -270,7 +309,6 @@ EvalState::EvalState(const Strings & _searchPath, DeterministicEvaluationMode ev , sFunctor(symbols.create("__functor")) , sToString(symbols.create("__toString")) , evalMode(evalMode) - , recordFileName(evalModeFile) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) { @@ -301,92 +339,42 @@ void EvalState::initializeDeterministicEvaluationMode() const char * playbackMode = getenv("NIX_PLAYBACK"); if (recordMode && !playbackMode) { evalMode = Record; - recordFileName = recordMode; } else if (playbackMode && !recordMode) { evalMode = Playback; - recordFileName = playbackMode; } else if (!playbackMode && !recordMode) { evalMode = Normal; } else throw EvalError("can't use both NIX_RECORDING and NIX_PLAYBACK"); } - if (evalMode == Playback || evalMode == Record) - assert(recordFileName != 0); - - if (evalMode == Playback) { - initializePlayback(); - } if (evalMode == Record || evalMode == Playback) { std::cerr << "Running in deterministic evaluation mode: " << evalMode << std::endl; } } -void EvalState::setRecordingInfo(bool fromArgs, std::map autoArgs_, Strings attrPaths, Strings files, string currentDir) +void EvalState::setRecordingInfo(Expr * e) { - recordingExpression.fromArgs = fromArgs; - recordingExpression.autoArgs = autoArgs_; - recordingExpression.attrPaths = attrPaths; - recordingExpression.files = files; - recordingExpression.currentDir = currentDir; + recordingExpression = e; } -void EvalState::getRecordingInfo(bool & fromArgs, std::map & autoArgs_, Strings & attrPaths, Strings & files, string & currentDir) +string EvalState::parameterValue(Value& value) { - if (!autoArgs_.empty()) { - throw Error("you can't supply auto arguments in playback mode"); - } - if (!attrPaths.empty()) { - throw Error("you can't supply attribute paths in playback mode"); - } - if (!files.empty()) { - throw Error("you can't supply files or expressions in playback mode"); - } - if (fromArgs) { - throw Error("--expr is invalid in playback mode"); - } - Symbol expressionS(symbols.create("expression")), - fromArgsS(symbols.create("fromArgs")), - autoArgsS(symbols.create("autoArgs")), - attributesS(symbols.create("attributes")), - filesS(symbols.create("files")), - currentDirS(symbols.create("currentDir")); - Value expression, fromArgsV, autoArgsV, attributesV, filesV, currentDirV; - getAttr(*playbackJson, expressionS, expression); - getAttr(expression, fromArgsS, fromArgsV); - getAttr(expression, autoArgsS, autoArgsV); - getAttr(expression, attributesS, attributesV); - getAttr(expression, filesS, filesV); - getAttr(expression, currentDirS, currentDirV); - fromArgs = forceBool(fromArgsV); - forceAttrs(autoArgsV); - for (auto it = autoArgsV.attrs->begin(); it != autoArgsV.attrs->end(); ++it) { - forceString(*it->value); - autoArgs_[it->name] = forceStringNoCtx(*it->value); - } - forceList(attributesV); - for (unsigned int i = 0; i < attributesV.listSize(); ++i) { - attrPaths.push_back(forceStringNoCtx(*attributesV.listElems()[i])); - } - forceList(filesV); - for (unsigned int i = 0; i < filesV.listSize(); ++i) { - files.push_back(forceStringNoCtx(*filesV.listElems()[i])); - } - currentDir = forceStringNoCtx(currentDirV); + std::stringstream result; + forceValueDeep(value); + result << *valueToExpression(value); + return result.str(); } -void EvalState::initializePlayback() + +void EvalState::addPlaybackSubstitutions(Value & top) { Symbol functionsSymbol(symbols.create("functions")), nameSymbol(symbols.create("name")), parametersSymbol(symbols.create("parameters")), resultSymbol(symbols.create("result")), sourcesSymbol(symbols.create("sources")); - Value & top (*allocValue()); - playbackJson = ⊤ - - parseJSON(*this, readFile(recordFileName), top); + Value functions; getAttr(top, functionsSymbol, functions); forceList(functions); @@ -400,18 +388,50 @@ void EvalState::initializePlayback() std::list parameterList; forceList(parameters); for (unsigned int j = 0; j < parameters.listSize(); ++j) { - parameterList.push_back(valueToJSON(*parameters.listElems()[j], false)); + parameterList.push_back(parameterValue(*parameters.listElems()[j])); + } + + auto key = std::make_pair(nameString, parameterList); + auto currentPlaybackValue = recording.find(key); + if (currentPlaybackValue == recording.end()) { + recording[key] = result; + } + else { + if (!eqValues(recording[key], result)) { + std::stringstream primopApp; + primopApp << nameString << "("; + bool f = false; + for (auto param: parameterList) { + if (f) primopApp << ", "; + primopApp << param; + f = true; + } + primopApp << ")"; + std::stringstream result1, result2; + result1 << result; + result2 << recording[key]; + throw EvalError(format("playback of '%s' has multiple possible values: '%s' and '%s'") % + primopApp.str() % result1.str() % result2.str()); + } } - recording[std::make_pair(nameString, parameterList)] = result; } + Value sources; getAttr(top, sourcesSymbol, sources); forceAttrs(sources); + PathSet context; for (auto it = sources.attrs->begin(); it != sources.attrs->end(); ++it) { - srcToStoreForPlayback[it->name] = forceStringNoCtx(*it->value); + addPlaybackSource(it->name, coerceToPath(noPos, *it->value, context)); } } +void EvalState::addPlaybackSource(const Path& from, const Path& to) +{ + //TODO check for suffixes/prefixes in srcToStoreForPlayback + // and srcToStore and look out for consistency + srcToStoreForPlayback[from] = to; +} + EvalState::~EvalState() { fileEvalCache.clear(); @@ -443,69 +463,72 @@ void EvalState::writeRecordingIntoStore(Value & result) } +struct ExprUnparsed: public Expr { + std::string unparsed; + ExprUnparsed(const std::string & up) : unparsed(up) {} + + virtual void show(std::ostream& str) { + str << unparsed; + } +}; + void EvalState::finalizeRecording(Value & result) { - //TODO: write this in a more functional style - std::stringstream out; - PathSet context; - out << "{\"expression\":{\"fromArgs\":"; - out << (recordingExpression.fromArgs ? "true" : "false"); - out << ",\"autoArgs\":{"; - bool isThisTheFirstTime0 = true; - for (auto autoArg: recordingExpression.autoArgs) { - if (!isThisTheFirstTime0) out << ","; - isThisTheFirstTime0 = false; - escapeJSON (out, autoArg.first); - out << ":"; - escapeJSON(out, autoArg.second); - } - out << "},\"attributes\":["; - isThisTheFirstTime0 = true; - for (auto attr: recordingExpression.attrPaths) { - if (!isThisTheFirstTime0) out << ","; - isThisTheFirstTime0 = false; - escapeJSON(out, attr); - } - out << "],\"files\":["; - isThisTheFirstTime0 = true; - for (auto file: recordingExpression.files) { - if (!isThisTheFirstTime0) out << ","; - isThisTheFirstTime0 = false; - escapeJSON(out, recordingExpression.fromArgs ? file : resolveExprPath(lookupFileArg(*this, file))); - } - out << "],\"currentDir\":\"" << recordingExpression.currentDir << "\"},\"functions\": [\n"; - - bool isThisTheFirstTime = true; - for (auto kv : recording) { - if (!isThisTheFirstTime) out << ","; - isThisTheFirstTime = false; + Value & top = *allocValue(); + //Value & top = result; + mkAttrs(top, 2); + Value * sources = allocAttr(top, symbols.create("sources")); + Value * functions = allocAttr(top, symbols.create("functions")); + top.attrs->sort(); - out << "{ \"name\":"; - escapeJSON(out, kv.first.first); - out << ", \"parameters\": ["; - - bool isThisTheFirstTime2 = true; - for (auto parameter: kv.first.second) { - if (!isThisTheFirstTime2) out << ", "; - isThisTheFirstTime2 = false; - out << parameter; + mkList(*functions, recording.size()); + int j = 0; + for (auto kv : recording) { + Value & attrs = *allocValue(); + functions->listElems()[j] = &attrs; + mkAttrs(attrs, 3); + Value * name = allocAttr(attrs, symbols.create("name")); + mkString(*name, kv.first.first); + Value * parameters = allocAttr(attrs, symbols.create("parameters")); + mkList(*parameters, kv.first.second.size()); + auto iter = kv.first.second.begin(); + for (int s = 0; s < kv.first.second.size(); ++s) { + parameters->listElems()[s] = allocValue(); + mkThunk_(*parameters->listElems()[s], new ExprUnparsed(*iter)); + ++iter; } - out << "], \"result\": " << valueToJSON(kv.second, false) << "}\n"; + Value * result = allocAttr(attrs, symbols.create("result")); + forceValueDeep(kv.second); + *result = kv.second; + //*result = kv.second; + attrs.attrs->sort(); + ++j; } - - out << "], \"sources\": {"; - bool isThisTheFirstTime3 = true; + PathSet context; + mkAttrs(*sources, srcToStoreForPlayback.size()); for (auto path : srcToStoreForPlayback) { - if (!isThisTheFirstTime3) out << ", "; - isThisTheFirstTime3 = false; + Value * p = allocAttr(*sources, symbols.create(path.first)); + mkPath(*p, path.second.c_str()); + assert (isInStore(path.second)); context.insert(path.second); - escapeJSON(out, path.first); - out << ":"; - escapeJSON(out, path.second); } - out << "}}"; - mkString(result, out.str(), context); + + mkAttrs(result, 2); + + { + Expr * e = valueToExpression(top); + std::stringstream out; + out << *e; + mkString(*allocAttr(result, symbols.create("recording")), out.str(), context); + } + { + std::stringstream exp; + exp << *recordingExpression; + mkString(*allocAttr(result, symbols.create("expressions")), exp.str()); + } + + result.attrs->sort(); } Path EvalState::checkSourcePath(const Path & path_) @@ -1729,8 +1752,14 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path, bool ign return dstPath; } -Path EvalState::copyPathToStoreIfItsNotAlreadyThere(PathSet context, Path path) +Path EvalState::copyPathToStoreIfItsNotAlreadyThere(PathSet & context, Path path) { + // special-case derivation.nix + // this is used so early the paths aren't set up correctly yet + // also doesn't really make sense to use it + if (symbols.create(path) == sDerivationNix) { + return path; + } if (evalMode == Playback) { Path rest; while (srcToStoreForPlayback[path] == "") { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 04dd8f2c7aa..c3f425a4208 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -21,7 +21,8 @@ class EvalState; typedef enum { Normal, Record, - Playback + Playback, + RecordAndPlayback } DeterministicEvaluationMode; typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); @@ -101,28 +102,25 @@ private: SearchPath searchPath; DeterministicEvaluationMode evalMode; - const char * recordFileName; - Value * playbackJson; //TODO: use some other structure, maybe a hashmap //since comparing strings with long same prefixes is slow std::map>, Value> recording; - struct RecordingExpression { - bool fromArgs; - std::map autoArgs; - Strings attrPaths; - Strings files; - string currentDir; - RecordingExpression() : fromArgs(false) {} - } recordingExpression; + Expr * recordingExpression; public: + bool isInPlaybackMode() { + return evalMode == Playback || evalMode == RecordAndPlayback; + } + - EvalState(const Strings & _searchPath, DeterministicEvaluationMode evalMode = Normal, const char * evalModeFile = nullptr); + EvalState(const Strings & _searchPath, DeterministicEvaluationMode evalMode = Normal); ~EvalState(); void addToSearchPath(const string & s, bool warn = false); - + void addPlaybackSubstitutions(nix::Value& top); + void addPlaybackSource(const Path & from, const Path & to); + Path checkSourcePath(const Path & path); /* Parse a Nix expression from the specified file. */ @@ -146,6 +144,9 @@ public: /* Evaluate an expression to normal form, storing the result in value `v'. */ void eval(Expr * e, Value & v); + + // convert a value back to an expression + Expr * valueToExpression(const Value & v); /* Evaluation the expression, then verify that it has the expected type. */ @@ -215,9 +216,9 @@ private: unsigned int arity, PrimOpFun primOp); std::string valueToJSON(Value & value, bool copyToStore); + string parameterValue(Value & value); void getAttr(Value & top, const Symbol & arg2, Value & v); void initializeDeterministicEvaluationMode(); - void initializePlayback(); void finalizeRecording (Value & result); void writeRecordingIntoStore (Value & result); @@ -232,7 +233,7 @@ private: std::list argList; for (unsigned int i = 0; i < arity; i++) { if (useArgument(i)) { - argList.push_back(state.valueToJSON(*args[i], false)); + argList.push_back(state.parameterValue(*args[i])); } } primOp(state, pos, args, v); @@ -245,13 +246,18 @@ private: std::list argList; for (unsigned int i = 0; i < arity; i++) { if(useArgument(i)) { - argList.push_back(state.valueToJSON(*args[i], false)); + argList.push_back(state.parameterValue(*args[i])); } } auto result = state.recording.find(std::make_pair(name, argList)); if (result == state.recording.end()) { std::string errorMsg("wanted to call "); - errorMsg +=name; + errorMsg += name; + errorMsg += "("; + for (auto arg: argList) { + errorMsg += arg + ", "; + } + errorMsg += ")"; throw EvalError(errorMsg.c_str()); } else { @@ -294,8 +300,7 @@ private: } public: - void getRecordingInfo(bool & fromArgs, std::map & autoArgs_, Strings & attrPaths, Strings & files, string & currentDir); - void setRecordingInfo(bool fromArgs, std::map autoArgs_, Strings attrPaths, Strings files, string currentDir); + void setRecordingInfo(Expr *); void getBuiltin(const string & name, Value & v); @@ -377,7 +382,7 @@ private: void addToBaseEnv(const string & name, Value * v, Symbol sym); void addToBaseEnv(const string & name, Value * v); Value * getImpureConstantPrimop(); - Path copyPathToStoreIfItsNotAlreadyThere(PathSet context, Path path); + Path copyPathToStoreIfItsNotAlreadyThere(PathSet & context, Path path); }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 29fa1a284a1..754eec8d7e2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -10,6 +10,7 @@ #include "names.hh" #include "eval-inline.hh" #include "download.hh" +#include "attr-path.hh" #include #include @@ -390,6 +391,26 @@ static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Valu mkString(v, state.restricted ? "" : getEnv(name)); } +/* wrapper around findAlongAttrPath to allow immitating nix-instantiate inside nix */ +static void prim_findAlongAttrPath(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string attrPath = state.forceString(*args[0]); + state.forceAttrs(*args[1]); + Value & result = *findAlongAttrPath(state, attrPath, *args[1]->attrs, *args[2]); + state.forceValue(result); + v = result; +} + +/* wrapper around findAlongAttrPath to allow immitating nix-instantiate inside nix */ +static void prim_playback(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + if (!state.isInPlaybackMode()) { + throwEvalError("playback primop is only allowed in playback mode (at '%s')", pos); + } + state.addPlaybackSubstitutions(*args[0]); + state.forceValue(*args[1]); + v = *args[1]; +} /* Evaluate the first argument, then return the second argument. */ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1773,6 +1794,8 @@ void EvalState::createBaseEnv() addPrimOp("__addErrorContext", 2, prim_addErrorContext); addPrimOp("__tryEval", 1, prim_tryEval); addImpurePrimOp<__getEnv, 1, prim_getEnv>(); + addPrimOp("__findAlongAttrPath", 3, prim_findAlongAttrPath); + addPrimOp("__playback", 2, prim_playback); // Strictness addPrimOp("__seq", 2, prim_seq); @@ -1790,6 +1813,7 @@ void EvalState::createBaseEnv() addPrimOp("dirOf", 1, prim_dirOf); addImpurePrimOp<__readFile, 1, prim_readFile>(); addImpurePrimOp<__readDir, 1, prim_readDir>(); + // this really isn't impure but we don't want to record __nixPath addImpurePrimOp<__findFile, 2, prim_findFile, onlyPos<1>>(); // Creating files @@ -1874,6 +1898,8 @@ void EvalState::createBaseEnv() mkString(*allocAttr(*v2, symbols.create("prefix")), i.first); v2->attrs->sort(); } + // TODO this really is an impure constant but rely that the user + // doesn't use this directly but ueses __findFile instead addConstant("__nixPath", v); /* Now that we've added all primops, sort the `builtins' set, diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 17587a75a1c..aec8a375784 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -88,6 +88,17 @@ void processExpr(EvalState & state, const Strings & attrPaths, } } +Expr * processExprExpression(EvalState & state, Expr * expr, const string & attrPath, Bindings & autoArgs) { + Expr * findAlongAttrPath = new ExprVar(state.symbols.create("__findAlongAttrPath")); + Expr * param1 = new ExprString(state.symbols.create(attrPath)); + ExprAttrs * param2 = new ExprAttrs(); + for (auto binding: autoArgs) { + //Remark: the value is either a string or a thunk without environment, in both cases valueToExpression does the right thing + param2->attrs[binding.name] = ExprAttrs::AttrDef(state.valueToExpression(*binding.value), noPos); + } + return new ExprApp(new ExprApp(new ExprApp(findAlongAttrPath, param1), param2), expr); +} + int main(int argc, char * * argv) { @@ -101,7 +112,7 @@ int main(int argc, char * * argv) bool findFile = false; bool evalOnly = false; bool parseOnly = false; - Path playback; + bool playback = false; bool record = false; OutputKind outputKind = okPlain; bool xmlOutputSourceLocation = true; @@ -151,7 +162,7 @@ int main(int argc, char * * argv) else if (*arg == "--dry-run") settings.readOnlyMode = true; else if (*arg == "--playback") - playback = getArg(*arg, arg, end); + playback = true; else if (*arg == "--record") record = true; else if (*arg != "" && arg->at(0) == '-') @@ -167,7 +178,7 @@ int main(int argc, char * * argv) store = openStore(); DeterministicEvaluationMode mode = record ? Record : Normal; - if (!playback.empty()) { + if (playback) { if (mode == Normal) { mode = Playback; } @@ -175,41 +186,57 @@ int main(int argc, char * * argv) throw Error("can't specify both --playback and --record"); } - EvalState state(searchPath, mode, playback.c_str()); + EvalState state(searchPath, mode); state.repair = repair; string currentPath = absPath("."); - if (mode != Playback) { - if (attrPaths.empty()) attrPaths.push_back(""); - if (findFile) { - for (auto & i : files) { - Path p = state.findFile(i); - if (p == "") throw Error(format("unable to find ‘%1%’") % i); - std::cout << p << std::endl; - } - return; + if (attrPaths.empty()) attrPaths.push_back(""); + if (findFile) { + for (auto & i : files) { + Path p = state.findFile(i); + if (p == "") throw Error(format("unable to find ‘%1%’") % i); + std::cout << p << std::endl; } - state.setRecordingInfo(fromArgs, autoArgs_, attrPaths, files, currentPath); - } - else { - state.getRecordingInfo(fromArgs, autoArgs_, attrPaths, files, currentPath); + return; } - + Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + std::list< Expr * > expressions; if (readStdin) { - Expr * e = parseStdin(state); - processExpr(state, attrPaths, parseOnly, strict, autoArgs, - evalOnly, outputKind, xmlOutputSourceLocation, e); + expressions.push_back(parseStdin(state)); } else if (files.empty() && !fromArgs) files.push_back("./default.nix"); for (auto & i : files) { - Expr * e = fromArgs - ? state.parseExprFromString(i, currentPath) - : state.parseExprFromFile(resolveExprPath(lookupFileArg(state, i))); + Expr * e; + if (fromArgs) { + e = state.parseExprFromString(i, currentPath); + } + else { + Path p = resolveExprPath(lookupFileArg(state, i)); + e = new ExprApp(new ExprVar(state.symbols.create("import")), new ExprPath(p)); + if (playback) { + state.addPlaybackSource(p, p); + } + e->bindVars(state.staticBaseEnv); + } + expressions.push_back(e); + } + if (mode == Record) { + ExprList * recordExpr = new ExprList(); + for (auto e: expressions) { + for (auto attrPath: attrPaths) { + recordExpr->elems.push_back(processExprExpression(state, e, attrPath, autoArgs)); + } + } + if (recordExpr->elems.size() == 1) + state.setRecordingInfo(recordExpr->elems.front()); + else + state.setRecordingInfo(recordExpr); + } + for (auto e: expressions) { processExpr(state, attrPaths, parseOnly, strict, autoArgs, - evalOnly, outputKind, xmlOutputSourceLocation, e); + evalOnly, outputKind, xmlOutputSourceLocation, e); } - state.printStats(); }); } From ec0748ba1470e87cca60e411e0f31affb133937a Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Fri, 22 Jan 2016 22:01:37 +0000 Subject: [PATCH 19/20] when doing a recording, write to the nix store --- src/nix-instantiate/nix-instantiate.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index aec8a375784..ccdeb278730 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -163,8 +163,10 @@ int main(int argc, char * * argv) settings.readOnlyMode = true; else if (*arg == "--playback") playback = true; - else if (*arg == "--record") + else if (*arg == "--record") { record = true; + wantsReadWrite = true; + } else if (*arg != "" && arg->at(0) == '-') return false; else From c1eddb10c16b6b407ba1f9abab38076c5eb32c1d Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Sat, 23 Jan 2016 13:42:40 +0000 Subject: [PATCH 20/20] cleaning up, lower interference with normal nix-instantiate, save link to recorded source closure (not registered in garbage collection) --- src/libexpr/eval.cc | 19 ++----- src/libexpr/eval.hh | 10 ++-- src/nix-instantiate/nix-instantiate.cc | 72 ++++++++++++-------------- 3 files changed, 42 insertions(+), 59 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f006b5d0f56..a34ee8b27ad 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -9,6 +9,7 @@ #include "json-to-value.hh" #include "common-opts.hh" #include "get-drvs.hh" +#include "shared.hh" #include #include @@ -353,11 +354,6 @@ void EvalState::initializeDeterministicEvaluationMode() } } -void EvalState::setRecordingInfo(Expr * e) -{ - recordingExpression = e; -} - string EvalState::parameterValue(Value& value) { std::stringstream result; @@ -435,15 +431,9 @@ void EvalState::addPlaybackSource(const Path& from, const Path& to) EvalState::~EvalState() { fileEvalCache.clear(); - - if (evalMode == Record) { - Value result; - finalizeRecording(result); - writeRecordingIntoStore(result); - } } -void EvalState::writeRecordingIntoStore(Value & result) +Path EvalState::writeRecordingIntoStore(Value & result, bool buildStorePath) { Value v; string path = settings.nixDataDir + "/nix/corepkgs/reproducable-derivation.nix"; @@ -456,10 +446,11 @@ void EvalState::writeRecordingIntoStore(Value & result) Bindings * autoArgs = allocBindings(4); getDerivations(*this, app, "", *autoArgs, drvs, false); Path drvPath = drvs.front().queryDrvPath(); + if (!buildStorePath) return drvPath; PathSet paths; paths.insert(drvPath); store->buildPaths(paths); - std::cerr << "succesfully build source closure: " << drvs.front().queryOutPath() << std::endl; + return drvs.front().queryOutPath(); } @@ -472,7 +463,7 @@ struct ExprUnparsed: public Expr { } }; -void EvalState::finalizeRecording(Value & result) +void EvalState::finalizeRecording(Value & result, Expr * recordingExpression) { Value & top = *allocValue(); //Value & top = result; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index c3f425a4208..49512cbc395 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -106,7 +106,6 @@ private: //TODO: use some other structure, maybe a hashmap //since comparing strings with long same prefixes is slow std::map>, Value> recording; - Expr * recordingExpression; public: bool isInPlaybackMode() { @@ -126,7 +125,9 @@ public: /* Parse a Nix expression from the specified file. */ Expr * parseExprFromFile(const Path & path); Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv); - + Expr * parseExprFromFileWithoutRecording(const Path & path); + Expr * parseExprFromFileWithoutRecording(const Path & path, StaticEnv &); + /* Parse a Nix expression from the specified string. */ Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv); Expr * parseExprFromString(const string & s, const Path & basePath); @@ -219,8 +220,6 @@ private: string parameterValue(Value & value); void getAttr(Value & top, const Symbol & arg2, Value & v); void initializeDeterministicEvaluationMode(); - void finalizeRecording (Value & result); - void writeRecordingIntoStore (Value & result); static bool constTrue(unsigned int arg) { return true; } template< int argumentPos > @@ -300,7 +299,8 @@ private: } public: - void setRecordingInfo(Expr *); + void finalizeRecording (Value & result, Expr * recordingExpressions); + Path writeRecordingIntoStore (Value & result, bool buildStorePath); void getBuiltin(const string & name, Value & v); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index ccdeb278730..a95499cf443 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -113,7 +113,7 @@ int main(int argc, char * * argv) bool evalOnly = false; bool parseOnly = false; bool playback = false; - bool record = false; + string record; OutputKind outputKind = okPlain; bool xmlOutputSourceLocation = true; bool strict = false; @@ -164,8 +164,8 @@ int main(int argc, char * * argv) else if (*arg == "--playback") playback = true; else if (*arg == "--record") { - record = true; wantsReadWrite = true; + record = getArg(*arg, arg, end); } else if (*arg != "" && arg->at(0) == '-') return false; @@ -179,20 +179,19 @@ int main(int argc, char * * argv) store = openStore(); - DeterministicEvaluationMode mode = record ? Record : Normal; + DeterministicEvaluationMode mode = record.empty() ? Normal : Record; if (playback) { - if (mode == Normal) { - mode = Playback; - } - else - throw Error("can't specify both --playback and --record"); + if (mode != Normal) throw Error ("--record and --playback exclude each other at the moment"); + mode = Playback; } EvalState state(searchPath, mode); state.repair = repair; - string currentPath = absPath("."); - + + Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + if (attrPaths.empty()) attrPaths.push_back(""); + if (findFile) { for (auto & i : files) { Path p = state.findFile(i); @@ -201,43 +200,36 @@ int main(int argc, char * * argv) } return; } + + ExprList * expressions = new ExprList(); - Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); - std::list< Expr * > expressions; if (readStdin) { - expressions.push_back(parseStdin(state)); + Expr * e = parseStdin(state); + for (auto p: attrPaths) + expressions->elems.push_back(processExprExpression(state, e, p, autoArgs)); + processExpr(state, attrPaths, parseOnly, strict, autoArgs, + evalOnly, outputKind, xmlOutputSourceLocation, e); } else if (files.empty() && !fromArgs) files.push_back("./default.nix"); + for (auto & i : files) { - Expr * e; - if (fromArgs) { - e = state.parseExprFromString(i, currentPath); - } - else { - Path p = resolveExprPath(lookupFileArg(state, i)); - e = new ExprApp(new ExprVar(state.symbols.create("import")), new ExprPath(p)); - if (playback) { - state.addPlaybackSource(p, p); - } - e->bindVars(state.staticBaseEnv); - } - expressions.push_back(e); + Expr * e = fromArgs + ? state.parseExprFromString(i, absPath(".")) + : state.parseExprFromFileWithoutRecording(resolveExprPath(lookupFileArg(state, i))); + for (auto p: attrPaths) + expressions->elems.push_back(processExprExpression(state, e, p, autoArgs)); + processExpr(state, attrPaths, parseOnly, strict, autoArgs, + evalOnly, outputKind, xmlOutputSourceLocation, e); } + + Expr * playbackExpression = expressions->elems.size() == 1 ? expressions->elems.front() : expressions; + if (mode == Record) { - ExprList * recordExpr = new ExprList(); - for (auto e: expressions) { - for (auto attrPath: attrPaths) { - recordExpr->elems.push_back(processExprExpression(state, e, attrPath, autoArgs)); - } - } - if (recordExpr->elems.size() == 1) - state.setRecordingInfo(recordExpr->elems.front()); - else - state.setRecordingInfo(recordExpr); - } - for (auto e: expressions) { - processExpr(state, attrPaths, parseOnly, strict, autoArgs, - evalOnly, outputKind, xmlOutputSourceLocation, e); + Value result; + state.finalizeRecording(result, playbackExpression); + bool drv = record.compare(record.length()-4, 4, ".drv"); + Path drvPath = state.writeRecordingIntoStore(result, drv); + createSymlink(drvPath, absPath(record)); } state.printStats(); });