Skip to content

Commit

Permalink
Merge branch 'master' into nix-next
Browse files Browse the repository at this point in the history
  • Loading branch information
@Ericson2314
Ericson2314 committed Jan 23, 2024
2 parents 20b0ad3 + d02e20a commit 9a86da0
Show file tree
Hide file tree
Showing 5 changed files with 68 additions and 42 deletions.
18 changes: 9 additions & 9 deletions src/hydra-queue-runner/build-remote.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ static Strings extraStoreArgs(std::string & machine)
}

static std::unique_ptr<SSHMaster::Connection> openConnection(
Machine::ptr machine, SSHMaster & master)
::Machine::ptr machine, SSHMaster & master)
{
Strings command = {"nix-store", "--serve", "--write"};
if (machine->isLocalhost()) {
Expand All @@ -61,7 +61,7 @@ static std::unique_ptr<SSHMaster::Connection> openConnection(


static void copyClosureTo(
Machine::Connection & conn,
::Machine::Connection & conn,
Store & destStore,
const StorePathSet & paths,
SubstituteFlag useSubstitutes = NoSubstitute)
Expand Down Expand Up @@ -148,7 +148,7 @@ static BasicDerivation sendInputs(
Step & step,
Store & localStore,
Store & destStore,
Machine::Connection & conn,
::Machine::Connection & conn,
unsigned int & overhead,
counter & nrStepsWaiting,
counter & nrStepsCopyingTo
Expand Down Expand Up @@ -204,7 +204,7 @@ static BasicDerivation sendInputs(
}

static BuildResult performBuild(
Machine::Connection & conn,
::Machine::Connection & conn,
Store & localStore,
StorePath drvPath,
const BasicDerivation & drv,
Expand Down Expand Up @@ -239,7 +239,7 @@ static BuildResult performBuild(
}

static std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(
Machine::Connection & conn,
::Machine::Connection & conn,
Store & localStore,
StorePathSet & outputs,
size_t & totalNarSize
Expand All @@ -265,7 +265,7 @@ static std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(
}

static void copyPathFromRemote(
Machine::Connection & conn,
::Machine::Connection & conn,
NarMemberDatas & narMembers,
Store & localStore,
Store & destStore,
Expand Down Expand Up @@ -295,7 +295,7 @@ static void copyPathFromRemote(
}

static void copyPathsFromRemote(
Machine::Connection & conn,
::Machine::Connection & conn,
NarMemberDatas & narMembers,
Store & localStore,
Store & destStore,
Expand Down Expand Up @@ -374,7 +374,7 @@ void RemoteResult::updateWithBuildResult(const nix::BuildResult & buildResult)


void State::buildRemote(ref<Store> destStore,
Machine::ptr machine, Step::ptr step,
::Machine::ptr machine, Step::ptr step,
const ServeProto::BuildOptions & buildOptions,
RemoteResult & result, std::shared_ptr<ActiveStep> activeStep,
std::function<void(StepState)> updateStep,
Expand Down Expand Up @@ -420,7 +420,7 @@ void State::buildRemote(ref<Store> destStore,
process. Meh. */
});

Machine::Connection conn {
::Machine::Connection conn {
{
.to = child->in.get(),
.from = child->out.get(),
Expand Down
2 changes: 1 addition & 1 deletion src/hydra-queue-runner/builder.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,7 @@ void State::failStep(
Step::ptr step,
BuildID buildId,
const RemoteResult & result,
Machine::ptr machine,
::Machine::ptr machine,
bool & stepFinished)
{
/* Register failure in the database for all Build objects that
Expand Down
10 changes: 5 additions & 5 deletions src/hydra-queue-runner/dispatcher.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ system_time State::doDispatch()
filter out temporarily disabled machines. */
struct MachineInfo
{
Machine::ptr machine;
::Machine::ptr machine;
unsigned long currentJobs;
};
std::vector<MachineInfo> machinesSorted;
Expand Down Expand Up @@ -231,11 +231,11 @@ system_time State::doDispatch()
sort(machinesSorted.begin(), machinesSorted.end(),
[](const MachineInfo & a, const MachineInfo & b) -> bool
{
float ta = std::round(a.currentJobs / a.machine->speedFactor);
float tb = std::round(b.currentJobs / b.machine->speedFactor);
float ta = std::round(a.currentJobs / a.machine->speedFactorFloat);
float tb = std::round(b.currentJobs / b.machine->speedFactorFloat);
return
ta != tb ? ta < tb :
a.machine->speedFactor != b.machine->speedFactor ? a.machine->speedFactor > b.machine->speedFactor :
a.machine->speedFactorFloat != b.machine->speedFactorFloat ? a.machine->speedFactorFloat > b.machine->speedFactorFloat :
a.currentJobs > b.currentJobs;
});

Expand Down Expand Up @@ -435,7 +435,7 @@ void Jobset::pruneSteps()
}


State::MachineReservation::MachineReservation(State & state, Step::ptr step, Machine::ptr machine)
State::MachineReservation::MachineReservation(State & state, Step::ptr step, ::Machine::ptr machine)
: state(state), step(step), machine(machine)
{
machine->state->currentJobs++;
Expand Down
59 changes: 40 additions & 19 deletions src/hydra-queue-runner/hydra-queue-runner.cc
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#include <iostream>
#include <thread>
#include <optional>
#include <type_traits>

#include <sys/types.h>
#include <sys/stat.h>
Expand Down Expand Up @@ -140,23 +141,43 @@ void State::parseMachines(const std::string & contents)
if (tokens.size() < 3) continue;
tokens.resize(8);

auto machine = std::make_shared<Machine>();
machine->sshName = tokens[0];
machine->systemTypes = tokenizeString<StringSet>(tokens[1], ",");
machine->sshKey = tokens[2] == "-" ? std::string("") : tokens[2];
if (tokens[3] != "")
machine->maxJobs = string2Int<decltype(machine->maxJobs)>(tokens[3]).value();
else
machine->maxJobs = 1;
machine->speedFactor = atof(tokens[4].c_str());
if (tokens[5] == "-") tokens[5] = "";
machine->supportedFeatures = tokenizeString<StringSet>(tokens[5], ",");
auto supportedFeatures = tokenizeString<StringSet>(tokens[5], ",");

if (tokens[6] == "-") tokens[6] = "";
machine->mandatoryFeatures = tokenizeString<StringSet>(tokens[6], ",");
for (auto & f : machine->mandatoryFeatures)
machine->supportedFeatures.insert(f);
if (tokens[7] != "" && tokens[7] != "-")
machine->sshPublicHostKey = base64Decode(tokens[7]);
auto mandatoryFeatures = tokenizeString<StringSet>(tokens[6], ",");

for (auto & f : mandatoryFeatures)
supportedFeatures.insert(f);

using MaxJobs = std::remove_const<decltype(nix::Machine::maxJobs)>::type;

auto machine = std::make_shared<::Machine>(nix::Machine {
// `storeUri`, not yet used
"",
// `systemTypes`, not yet used
{},
// `sshKey`
tokens[2] == "-" ? "" : tokens[2],
// `maxJobs`
tokens[3] != ""
? string2Int<MaxJobs>(tokens[3]).value()
: 1,
// `speedFactor`, not yet used
1,
// `supportedFeatures`
std::move(supportedFeatures),
// `mandatoryFeatures`
std::move(mandatoryFeatures),
// `sshPublicHostKey`
tokens[7] != "" && tokens[7] != "-"
? base64Decode(tokens[7])
: "",
});

machine->sshName = tokens[0];
machine->systemTypesSet = tokenizeString<StringSet>(tokens[1], ",");
machine->speedFactorFloat = atof(tokens[4].c_str());

/* Re-use the State object of the previous machine with the
same name. */
Expand All @@ -166,7 +187,7 @@ void State::parseMachines(const std::string & contents)
else
printMsg(lvlChatty, "updating machine ‘%1%’", machine->sshName);
machine->state = i == oldMachines.end()
? std::make_shared<Machine::State>()
? std::make_shared<::Machine::State>()
: i->second->state;
newMachines[machine->sshName] = machine;
}
Expand All @@ -175,9 +196,9 @@ void State::parseMachines(const std::string & contents)
if (newMachines.find(m.first) == newMachines.end()) {
if (m.second->enabled)
printInfo("removing machine ‘%1%’", m.first);
/* Add a disabled Machine object to make sure stats are
/* Add a disabled ::Machine object to make sure stats are
maintained. */
auto machine = std::make_shared<Machine>(*(m.second));
auto machine = std::make_shared<::Machine>(*(m.second));
machine->enabled = false;
newMachines[m.first] = machine;
}
Expand Down Expand Up @@ -596,7 +617,7 @@ void State::dumpStatus(Connection & conn)

json machine = {
{"enabled", m->enabled},
{"systemTypes", m->systemTypes},
{"systemTypes", m->systemTypesSet},
{"supportedFeatures", m->supportedFeatures},
{"mandatoryFeatures", m->mandatoryFeatures},
{"nrStepsDone", s->nrStepsDone.load()},
Expand Down
21 changes: 13 additions & 8 deletions src/hydra-queue-runner/state.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include "nar-extractor.hh"
#include "serve-protocol.hh"
#include "serve-protocol-impl.hh"
#include "machines.hh"


typedef unsigned int BuildID;
Expand Down Expand Up @@ -235,17 +236,21 @@ void getDependents(Step::ptr step, std::set<Build::ptr> & builds, std::set<Step:
void visitDependencies(std::function<void(Step::ptr)> visitor, Step::ptr step);


struct Machine
struct Machine : nix::Machine
{
typedef std::shared_ptr<Machine> ptr;

bool enabled{true};
/* TODO Get rid of: `nix::Machine::storeUri` is normalized in a way
we are not yet used to, but once we are, we don't need this. */
std::string sshName;

std::string sshName, sshKey;
std::set<std::string> systemTypes, supportedFeatures, mandatoryFeatures;
unsigned int maxJobs = 1;
float speedFactor = 1.0;
std::string sshPublicHostKey;
/* TODO Get rid once `nix::Machine::systemTypes` is a set not
vector. */
std::set<std::string> systemTypesSet;

/* TODO Get rid once `nix::Machine::systemTypes` is a `float` not
an `int`. */
float speedFactorFloat = 1.0;

struct State {
typedef std::shared_ptr<State> ptr;
Expand Down Expand Up @@ -273,7 +278,7 @@ struct Machine
{
/* Check that this machine is of the type required by the
step. */
if (!systemTypes.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform))
if (!systemTypesSet.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform))
return false;

/* Check that the step requires all mandatory features of this
Expand Down

0 comments on commit 9a86da0

Please sign in to comment.