diff --git a/corepkgs/reproducable-derivation.nix b/corepkgs/reproducable-derivation.nix index 313a7f34565..0cb8c1d6478 100644 --- a/corepkgs/reproducable-derivation.nix +++ b/corepkgs/reproducable-derivation.nix @@ -1,4 +1,4 @@ -result: +{ recording, expressions}: with import ./config.nix; derivation { name = "source-closure"; @@ -6,7 +6,10 @@ derivation { args = ["-e" (__toFile "name" '' #!/bin/bash ${coreutils}/mkdir -p $out/nix-support - ${coreutils}/cp ${__toFile "result" result} $out/nix-support/source + ${coreutils}/cp ${__toFile "deterministic-recording" recording} $out/nix-support/recording.nix + ${coreutils}/cp ${__toFile "expressions" expressions} $out/nix-support/expressions.nix + echo "__playback { sources = { \"$out\" = ./.; }; functions = []; }" > $out/default.nix + echo "(__playback (import ./nix-support/recording.nix) (import ./nix-support/expressions.nix))" >> $out/default.nix '')]; system = builtins.currentSystem; } \ No newline at end of file diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7a8b54440a2..f006b5d0f56 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -142,6 +142,45 @@ static void printValue(std::ostream & str, std::set & active, con active.erase(&v); } +Expr * EvalState::valueToExpression(const Value & v) +{ + switch (v.type) { + case tInt: + return new ExprInt(v.integer); + case tBool: + return new ExprVar(symbols.create(v.boolean ? "true" : "false")); + case tString: + return new ExprString(symbols.create(v.string.s)); + case tPath: + return new ExprPath(v.path); + case tNull: + return new ExprVar(symbols.create("null")); + case tAttrs: + { + ExprAttrs * result = new ExprAttrs(); + for (auto a: *v.attrs) { + result->attrs.insert(std::make_pair(a.name, ExprAttrs::AttrDef(valueToExpression(*a.value), *a.pos))); + } + return result; + } + case tList1: + case tList2: + case tListN: + { + ExprList * result = new ExprList(); + for (int i = 0; i < v.listSize(); ++i) { + result->elems.push_back(valueToExpression(*v.listElems()[i])); + } + return result; + } + case tThunk: + return v.thunk.expr; + default: + throw Error("value not supported while converting"); + } +} + + std::ostream & operator << (std::ostream & str, const Value & v) { @@ -251,7 +290,7 @@ static Strings parseNixPath(const string & in) } -EvalState::EvalState(const Strings & _searchPath, DeterministicEvaluationMode evalMode, const char * evalModeFile) +EvalState::EvalState(const Strings & _searchPath, DeterministicEvaluationMode evalMode) : sWith(symbols.create("")) , sOutPath(symbols.create("outPath")) , sDrvPath(symbols.create("drvPath")) @@ -270,7 +309,6 @@ EvalState::EvalState(const Strings & _searchPath, DeterministicEvaluationMode ev , sFunctor(symbols.create("__functor")) , sToString(symbols.create("__toString")) , evalMode(evalMode) - , recordFileName(evalModeFile) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) { @@ -301,92 +339,42 @@ void EvalState::initializeDeterministicEvaluationMode() const char * playbackMode = getenv("NIX_PLAYBACK"); if (recordMode && !playbackMode) { evalMode = Record; - recordFileName = recordMode; } else if (playbackMode && !recordMode) { evalMode = Playback; - recordFileName = playbackMode; } else if (!playbackMode && !recordMode) { evalMode = Normal; } else throw EvalError("can't use both NIX_RECORDING and NIX_PLAYBACK"); } - if (evalMode == Playback || evalMode == Record) - assert(recordFileName != 0); - - if (evalMode == Playback) { - initializePlayback(); - } if (evalMode == Record || evalMode == Playback) { std::cerr << "Running in deterministic evaluation mode: " << evalMode << std::endl; } } -void EvalState::setRecordingInfo(bool fromArgs, std::map autoArgs_, Strings attrPaths, Strings files, string currentDir) +void EvalState::setRecordingInfo(Expr * e) { - recordingExpression.fromArgs = fromArgs; - recordingExpression.autoArgs = autoArgs_; - recordingExpression.attrPaths = attrPaths; - recordingExpression.files = files; - recordingExpression.currentDir = currentDir; + recordingExpression = e; } -void EvalState::getRecordingInfo(bool & fromArgs, std::map & autoArgs_, Strings & attrPaths, Strings & files, string & currentDir) +string EvalState::parameterValue(Value& value) { - if (!autoArgs_.empty()) { - throw Error("you can't supply auto arguments in playback mode"); - } - if (!attrPaths.empty()) { - throw Error("you can't supply attribute paths in playback mode"); - } - if (!files.empty()) { - throw Error("you can't supply files or expressions in playback mode"); - } - if (fromArgs) { - throw Error("--expr is invalid in playback mode"); - } - Symbol expressionS(symbols.create("expression")), - fromArgsS(symbols.create("fromArgs")), - autoArgsS(symbols.create("autoArgs")), - attributesS(symbols.create("attributes")), - filesS(symbols.create("files")), - currentDirS(symbols.create("currentDir")); - Value expression, fromArgsV, autoArgsV, attributesV, filesV, currentDirV; - getAttr(*playbackJson, expressionS, expression); - getAttr(expression, fromArgsS, fromArgsV); - getAttr(expression, autoArgsS, autoArgsV); - getAttr(expression, attributesS, attributesV); - getAttr(expression, filesS, filesV); - getAttr(expression, currentDirS, currentDirV); - fromArgs = forceBool(fromArgsV); - forceAttrs(autoArgsV); - for (auto it = autoArgsV.attrs->begin(); it != autoArgsV.attrs->end(); ++it) { - forceString(*it->value); - autoArgs_[it->name] = forceStringNoCtx(*it->value); - } - forceList(attributesV); - for (unsigned int i = 0; i < attributesV.listSize(); ++i) { - attrPaths.push_back(forceStringNoCtx(*attributesV.listElems()[i])); - } - forceList(filesV); - for (unsigned int i = 0; i < filesV.listSize(); ++i) { - files.push_back(forceStringNoCtx(*filesV.listElems()[i])); - } - currentDir = forceStringNoCtx(currentDirV); + std::stringstream result; + forceValueDeep(value); + result << *valueToExpression(value); + return result.str(); } -void EvalState::initializePlayback() + +void EvalState::addPlaybackSubstitutions(Value & top) { Symbol functionsSymbol(symbols.create("functions")), nameSymbol(symbols.create("name")), parametersSymbol(symbols.create("parameters")), resultSymbol(symbols.create("result")), sourcesSymbol(symbols.create("sources")); - Value & top (*allocValue()); - playbackJson = ⊤ - - parseJSON(*this, readFile(recordFileName), top); + Value functions; getAttr(top, functionsSymbol, functions); forceList(functions); @@ -400,18 +388,50 @@ void EvalState::initializePlayback() std::list parameterList; forceList(parameters); for (unsigned int j = 0; j < parameters.listSize(); ++j) { - parameterList.push_back(valueToJSON(*parameters.listElems()[j], false)); + parameterList.push_back(parameterValue(*parameters.listElems()[j])); + } + + auto key = std::make_pair(nameString, parameterList); + auto currentPlaybackValue = recording.find(key); + if (currentPlaybackValue == recording.end()) { + recording[key] = result; + } + else { + if (!eqValues(recording[key], result)) { + std::stringstream primopApp; + primopApp << nameString << "("; + bool f = false; + for (auto param: parameterList) { + if (f) primopApp << ", "; + primopApp << param; + f = true; + } + primopApp << ")"; + std::stringstream result1, result2; + result1 << result; + result2 << recording[key]; + throw EvalError(format("playback of '%s' has multiple possible values: '%s' and '%s'") % + primopApp.str() % result1.str() % result2.str()); + } } - recording[std::make_pair(nameString, parameterList)] = result; } + Value sources; getAttr(top, sourcesSymbol, sources); forceAttrs(sources); + PathSet context; for (auto it = sources.attrs->begin(); it != sources.attrs->end(); ++it) { - srcToStoreForPlayback[it->name] = forceStringNoCtx(*it->value); + addPlaybackSource(it->name, coerceToPath(noPos, *it->value, context)); } } +void EvalState::addPlaybackSource(const Path& from, const Path& to) +{ + //TODO check for suffixes/prefixes in srcToStoreForPlayback + // and srcToStore and look out for consistency + srcToStoreForPlayback[from] = to; +} + EvalState::~EvalState() { fileEvalCache.clear(); @@ -443,69 +463,72 @@ void EvalState::writeRecordingIntoStore(Value & result) } +struct ExprUnparsed: public Expr { + std::string unparsed; + ExprUnparsed(const std::string & up) : unparsed(up) {} + + virtual void show(std::ostream& str) { + str << unparsed; + } +}; + void EvalState::finalizeRecording(Value & result) { - //TODO: write this in a more functional style - std::stringstream out; - PathSet context; - out << "{\"expression\":{\"fromArgs\":"; - out << (recordingExpression.fromArgs ? "true" : "false"); - out << ",\"autoArgs\":{"; - bool isThisTheFirstTime0 = true; - for (auto autoArg: recordingExpression.autoArgs) { - if (!isThisTheFirstTime0) out << ","; - isThisTheFirstTime0 = false; - escapeJSON (out, autoArg.first); - out << ":"; - escapeJSON(out, autoArg.second); - } - out << "},\"attributes\":["; - isThisTheFirstTime0 = true; - for (auto attr: recordingExpression.attrPaths) { - if (!isThisTheFirstTime0) out << ","; - isThisTheFirstTime0 = false; - escapeJSON(out, attr); - } - out << "],\"files\":["; - isThisTheFirstTime0 = true; - for (auto file: recordingExpression.files) { - if (!isThisTheFirstTime0) out << ","; - isThisTheFirstTime0 = false; - escapeJSON(out, recordingExpression.fromArgs ? file : resolveExprPath(lookupFileArg(*this, file))); - } - out << "],\"currentDir\":\"" << recordingExpression.currentDir << "\"},\"functions\": [\n"; - - bool isThisTheFirstTime = true; - for (auto kv : recording) { - if (!isThisTheFirstTime) out << ","; - isThisTheFirstTime = false; + Value & top = *allocValue(); + //Value & top = result; + mkAttrs(top, 2); + Value * sources = allocAttr(top, symbols.create("sources")); + Value * functions = allocAttr(top, symbols.create("functions")); + top.attrs->sort(); - out << "{ \"name\":"; - escapeJSON(out, kv.first.first); - out << ", \"parameters\": ["; - - bool isThisTheFirstTime2 = true; - for (auto parameter: kv.first.second) { - if (!isThisTheFirstTime2) out << ", "; - isThisTheFirstTime2 = false; - out << parameter; + mkList(*functions, recording.size()); + int j = 0; + for (auto kv : recording) { + Value & attrs = *allocValue(); + functions->listElems()[j] = &attrs; + mkAttrs(attrs, 3); + Value * name = allocAttr(attrs, symbols.create("name")); + mkString(*name, kv.first.first); + Value * parameters = allocAttr(attrs, symbols.create("parameters")); + mkList(*parameters, kv.first.second.size()); + auto iter = kv.first.second.begin(); + for (int s = 0; s < kv.first.second.size(); ++s) { + parameters->listElems()[s] = allocValue(); + mkThunk_(*parameters->listElems()[s], new ExprUnparsed(*iter)); + ++iter; } - out << "], \"result\": " << valueToJSON(kv.second, false) << "}\n"; + Value * result = allocAttr(attrs, symbols.create("result")); + forceValueDeep(kv.second); + *result = kv.second; + //*result = kv.second; + attrs.attrs->sort(); + ++j; } - - out << "], \"sources\": {"; - bool isThisTheFirstTime3 = true; + PathSet context; + mkAttrs(*sources, srcToStoreForPlayback.size()); for (auto path : srcToStoreForPlayback) { - if (!isThisTheFirstTime3) out << ", "; - isThisTheFirstTime3 = false; + Value * p = allocAttr(*sources, symbols.create(path.first)); + mkPath(*p, path.second.c_str()); + assert (isInStore(path.second)); context.insert(path.second); - escapeJSON(out, path.first); - out << ":"; - escapeJSON(out, path.second); } - out << "}}"; - mkString(result, out.str(), context); + + mkAttrs(result, 2); + + { + Expr * e = valueToExpression(top); + std::stringstream out; + out << *e; + mkString(*allocAttr(result, symbols.create("recording")), out.str(), context); + } + { + std::stringstream exp; + exp << *recordingExpression; + mkString(*allocAttr(result, symbols.create("expressions")), exp.str()); + } + + result.attrs->sort(); } Path EvalState::checkSourcePath(const Path & path_) @@ -1729,8 +1752,14 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path, bool ign return dstPath; } -Path EvalState::copyPathToStoreIfItsNotAlreadyThere(PathSet context, Path path) +Path EvalState::copyPathToStoreIfItsNotAlreadyThere(PathSet & context, Path path) { + // special-case derivation.nix + // this is used so early the paths aren't set up correctly yet + // also doesn't really make sense to use it + if (symbols.create(path) == sDerivationNix) { + return path; + } if (evalMode == Playback) { Path rest; while (srcToStoreForPlayback[path] == "") { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 04dd8f2c7aa..c3f425a4208 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -21,7 +21,8 @@ class EvalState; typedef enum { Normal, Record, - Playback + Playback, + RecordAndPlayback } DeterministicEvaluationMode; typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); @@ -101,28 +102,25 @@ private: SearchPath searchPath; DeterministicEvaluationMode evalMode; - const char * recordFileName; - Value * playbackJson; //TODO: use some other structure, maybe a hashmap //since comparing strings with long same prefixes is slow std::map>, Value> recording; - struct RecordingExpression { - bool fromArgs; - std::map autoArgs; - Strings attrPaths; - Strings files; - string currentDir; - RecordingExpression() : fromArgs(false) {} - } recordingExpression; + Expr * recordingExpression; public: + bool isInPlaybackMode() { + return evalMode == Playback || evalMode == RecordAndPlayback; + } + - EvalState(const Strings & _searchPath, DeterministicEvaluationMode evalMode = Normal, const char * evalModeFile = nullptr); + EvalState(const Strings & _searchPath, DeterministicEvaluationMode evalMode = Normal); ~EvalState(); void addToSearchPath(const string & s, bool warn = false); - + void addPlaybackSubstitutions(nix::Value& top); + void addPlaybackSource(const Path & from, const Path & to); + Path checkSourcePath(const Path & path); /* Parse a Nix expression from the specified file. */ @@ -146,6 +144,9 @@ public: /* Evaluate an expression to normal form, storing the result in value `v'. */ void eval(Expr * e, Value & v); + + // convert a value back to an expression + Expr * valueToExpression(const Value & v); /* Evaluation the expression, then verify that it has the expected type. */ @@ -215,9 +216,9 @@ private: unsigned int arity, PrimOpFun primOp); std::string valueToJSON(Value & value, bool copyToStore); + string parameterValue(Value & value); void getAttr(Value & top, const Symbol & arg2, Value & v); void initializeDeterministicEvaluationMode(); - void initializePlayback(); void finalizeRecording (Value & result); void writeRecordingIntoStore (Value & result); @@ -232,7 +233,7 @@ private: std::list argList; for (unsigned int i = 0; i < arity; i++) { if (useArgument(i)) { - argList.push_back(state.valueToJSON(*args[i], false)); + argList.push_back(state.parameterValue(*args[i])); } } primOp(state, pos, args, v); @@ -245,13 +246,18 @@ private: std::list argList; for (unsigned int i = 0; i < arity; i++) { if(useArgument(i)) { - argList.push_back(state.valueToJSON(*args[i], false)); + argList.push_back(state.parameterValue(*args[i])); } } auto result = state.recording.find(std::make_pair(name, argList)); if (result == state.recording.end()) { std::string errorMsg("wanted to call "); - errorMsg +=name; + errorMsg += name; + errorMsg += "("; + for (auto arg: argList) { + errorMsg += arg + ", "; + } + errorMsg += ")"; throw EvalError(errorMsg.c_str()); } else { @@ -294,8 +300,7 @@ private: } public: - void getRecordingInfo(bool & fromArgs, std::map & autoArgs_, Strings & attrPaths, Strings & files, string & currentDir); - void setRecordingInfo(bool fromArgs, std::map autoArgs_, Strings attrPaths, Strings files, string currentDir); + void setRecordingInfo(Expr *); void getBuiltin(const string & name, Value & v); @@ -377,7 +382,7 @@ private: void addToBaseEnv(const string & name, Value * v, Symbol sym); void addToBaseEnv(const string & name, Value * v); Value * getImpureConstantPrimop(); - Path copyPathToStoreIfItsNotAlreadyThere(PathSet context, Path path); + Path copyPathToStoreIfItsNotAlreadyThere(PathSet & context, Path path); }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 29fa1a284a1..754eec8d7e2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -10,6 +10,7 @@ #include "names.hh" #include "eval-inline.hh" #include "download.hh" +#include "attr-path.hh" #include #include @@ -390,6 +391,26 @@ static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Valu mkString(v, state.restricted ? "" : getEnv(name)); } +/* wrapper around findAlongAttrPath to allow immitating nix-instantiate inside nix */ +static void prim_findAlongAttrPath(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string attrPath = state.forceString(*args[0]); + state.forceAttrs(*args[1]); + Value & result = *findAlongAttrPath(state, attrPath, *args[1]->attrs, *args[2]); + state.forceValue(result); + v = result; +} + +/* wrapper around findAlongAttrPath to allow immitating nix-instantiate inside nix */ +static void prim_playback(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + if (!state.isInPlaybackMode()) { + throwEvalError("playback primop is only allowed in playback mode (at '%s')", pos); + } + state.addPlaybackSubstitutions(*args[0]); + state.forceValue(*args[1]); + v = *args[1]; +} /* Evaluate the first argument, then return the second argument. */ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1773,6 +1794,8 @@ void EvalState::createBaseEnv() addPrimOp("__addErrorContext", 2, prim_addErrorContext); addPrimOp("__tryEval", 1, prim_tryEval); addImpurePrimOp<__getEnv, 1, prim_getEnv>(); + addPrimOp("__findAlongAttrPath", 3, prim_findAlongAttrPath); + addPrimOp("__playback", 2, prim_playback); // Strictness addPrimOp("__seq", 2, prim_seq); @@ -1790,6 +1813,7 @@ void EvalState::createBaseEnv() addPrimOp("dirOf", 1, prim_dirOf); addImpurePrimOp<__readFile, 1, prim_readFile>(); addImpurePrimOp<__readDir, 1, prim_readDir>(); + // this really isn't impure but we don't want to record __nixPath addImpurePrimOp<__findFile, 2, prim_findFile, onlyPos<1>>(); // Creating files @@ -1874,6 +1898,8 @@ void EvalState::createBaseEnv() mkString(*allocAttr(*v2, symbols.create("prefix")), i.first); v2->attrs->sort(); } + // TODO this really is an impure constant but rely that the user + // doesn't use this directly but ueses __findFile instead addConstant("__nixPath", v); /* Now that we've added all primops, sort the `builtins' set, diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 17587a75a1c..aec8a375784 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -88,6 +88,17 @@ void processExpr(EvalState & state, const Strings & attrPaths, } } +Expr * processExprExpression(EvalState & state, Expr * expr, const string & attrPath, Bindings & autoArgs) { + Expr * findAlongAttrPath = new ExprVar(state.symbols.create("__findAlongAttrPath")); + Expr * param1 = new ExprString(state.symbols.create(attrPath)); + ExprAttrs * param2 = new ExprAttrs(); + for (auto binding: autoArgs) { + //Remark: the value is either a string or a thunk without environment, in both cases valueToExpression does the right thing + param2->attrs[binding.name] = ExprAttrs::AttrDef(state.valueToExpression(*binding.value), noPos); + } + return new ExprApp(new ExprApp(new ExprApp(findAlongAttrPath, param1), param2), expr); +} + int main(int argc, char * * argv) { @@ -101,7 +112,7 @@ int main(int argc, char * * argv) bool findFile = false; bool evalOnly = false; bool parseOnly = false; - Path playback; + bool playback = false; bool record = false; OutputKind outputKind = okPlain; bool xmlOutputSourceLocation = true; @@ -151,7 +162,7 @@ int main(int argc, char * * argv) else if (*arg == "--dry-run") settings.readOnlyMode = true; else if (*arg == "--playback") - playback = getArg(*arg, arg, end); + playback = true; else if (*arg == "--record") record = true; else if (*arg != "" && arg->at(0) == '-') @@ -167,7 +178,7 @@ int main(int argc, char * * argv) store = openStore(); DeterministicEvaluationMode mode = record ? Record : Normal; - if (!playback.empty()) { + if (playback) { if (mode == Normal) { mode = Playback; } @@ -175,41 +186,57 @@ int main(int argc, char * * argv) throw Error("can't specify both --playback and --record"); } - EvalState state(searchPath, mode, playback.c_str()); + EvalState state(searchPath, mode); state.repair = repair; string currentPath = absPath("."); - if (mode != Playback) { - if (attrPaths.empty()) attrPaths.push_back(""); - if (findFile) { - for (auto & i : files) { - Path p = state.findFile(i); - if (p == "") throw Error(format("unable to find ‘%1%’") % i); - std::cout << p << std::endl; - } - return; + if (attrPaths.empty()) attrPaths.push_back(""); + if (findFile) { + for (auto & i : files) { + Path p = state.findFile(i); + if (p == "") throw Error(format("unable to find ‘%1%’") % i); + std::cout << p << std::endl; } - state.setRecordingInfo(fromArgs, autoArgs_, attrPaths, files, currentPath); - } - else { - state.getRecordingInfo(fromArgs, autoArgs_, attrPaths, files, currentPath); + return; } - + Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + std::list< Expr * > expressions; if (readStdin) { - Expr * e = parseStdin(state); - processExpr(state, attrPaths, parseOnly, strict, autoArgs, - evalOnly, outputKind, xmlOutputSourceLocation, e); + expressions.push_back(parseStdin(state)); } else if (files.empty() && !fromArgs) files.push_back("./default.nix"); for (auto & i : files) { - Expr * e = fromArgs - ? state.parseExprFromString(i, currentPath) - : state.parseExprFromFile(resolveExprPath(lookupFileArg(state, i))); + Expr * e; + if (fromArgs) { + e = state.parseExprFromString(i, currentPath); + } + else { + Path p = resolveExprPath(lookupFileArg(state, i)); + e = new ExprApp(new ExprVar(state.symbols.create("import")), new ExprPath(p)); + if (playback) { + state.addPlaybackSource(p, p); + } + e->bindVars(state.staticBaseEnv); + } + expressions.push_back(e); + } + if (mode == Record) { + ExprList * recordExpr = new ExprList(); + for (auto e: expressions) { + for (auto attrPath: attrPaths) { + recordExpr->elems.push_back(processExprExpression(state, e, attrPath, autoArgs)); + } + } + if (recordExpr->elems.size() == 1) + state.setRecordingInfo(recordExpr->elems.front()); + else + state.setRecordingInfo(recordExpr); + } + for (auto e: expressions) { processExpr(state, attrPaths, parseOnly, strict, autoArgs, - evalOnly, outputKind, xmlOutputSourceLocation, e); + evalOnly, outputKind, xmlOutputSourceLocation, e); } - state.printStats(); }); }