headless

#!/usr/bin/env node

require('./tests/numbas-runtime.js');
require('./tests/locales.js');
require('./tests/marking_scripts.js');
const fs = require('fs');
const path = require('path');
const {Console} = require('console');
const stream = require('stream');

var captured_log = '';
const real_console = new Console(process.stdout,process.stderr);
global.alert = function(m) { real_console.error(m); }
const s = new stream.Writable({write: function(chunk,enc,next) {
    captured_log += chunk;
    next();
}});
const fake_console = new Console(s,process.stderr);
Object.keys(console).forEach(function(n) {
    if(typeof console[n] == 'function' && typeof fake_console[n] == 'function') {
        console[n] = fake_console[n];
    }
});

var standard_reqs = ['json','jme','localisation','parts/numberentry','parts/jme','parts/matrixentry', 'parts/multipleresponse', 'parts/patternmatch','parts/gapfill','question']

class PartCreditError extends Error {
}
PartCreditError.prototype.originalMessages = ['headless.not full credit'];

function tick(correct) {
    return correct ? '✔' : '✖';
}

function die(e) {
    process.stdout.write(JSON.stringify({success:false, message: e.message, stack: e.stack, originalMessages: e.originalMessages || []}));
    end();
}
function succeed() {
    process.stdout.write(JSON.stringify({success:true}));
    end();
}
function end(code) {
    if(captured_log) {
        process.stderr.write(`------------\nConsole log:\n${captured_log}\n----------`);
    }
    process.exit(code);
}

function parse_numbasobject(data) {
    var re_header = /^\/\/ Numbas version: [^\n]*\n/;
    if(!data.match(re_header)) {
        throw(new Error("Not a valid .exam file"));
    }
    data = data.replace(re_header,'');
    return JSON.parse(data);
}

function test_question(q) {
    q.generateVariables();
    var promise = q.signals.on('ready').then(function() {
        return Promise.all(q.parts.map(p => test_part(p) ));
    }).catch(e=>{
        if(q.scope) {
            console.log("Question variables:");
            for(let name of Object.keys(q.scope.variables)) {
                const value = Numbas.jme.display.treeToJME({tok:q.scope.getVariable(name)});
                console.log(`  ${name}: ${value}`);
            }
        }
        die(e);
    });
    return promise;
}

function scope_for_part(p) {
    return p.settings.variableReplacementStrategy=='alwaysreplace' ? p.errorCarriedForwardScope() : p.getScope();
}

function answer_for_part(p) {
    if(p.type=='gapfill') {
        return p.gaps.map(answer_for_part);
    }
    var answer = p.getCorrectAnswer(scope_for_part(p));
    return answer;
}

async function test_part(part) {
    if(part.type=='information') {
        return;
    }
    if(part.type=='gapfill' && part.gaps.length==0) {
        return;
    }
    var answer = answer_for_part(part);
    part.storeAnswer(answer);
    part.setStudentAnswer();
    const scope = scope_for_part(part);
    part.submit();
    var promise;
    if(part.waiting_for_pre_submit) {
        promise = part.waiting_for_pre_submit.then(function() {
            part.submit();
        });
    } else {
        promise = Promise.resolve(true);
    }
    await promise;

    if(Numbas.math.precround(part.credit,5)!=1) {
        const alternatives_result = part.markAlternatives(part.getScope(), undefined, '');

        const niceName = Numbas.util.capitalise(Numbas.util.nicePartName(part.path));
        real_console.error(`✖ ${niceName}: ${part.type}`);
        real_console.error(`Answer: ${JSON.stringify(answer)}`);
        real_console.error(`Credit: ${part.credit}`);
        if(part.settings.variableReplacementStrategy=='alwaysreplace') {
            real_console.error('Variables:');
            for(let name of Object.keys(scope.variables)) {
                const value = Numbas.jme.display.treeToJME({tok:scope.getVariable(name)});
                real_console.error(`  ${name}: ${value}`);
            }
        }
        const state_errors = alternatives_result.result.script_result.state_errors;
        if(state_errors) {
            real_console.error(`State errors:`);
            for(let [state, error] of Object.entries(state_errors)) {
                real_console.error(`${state}: ${error}`);
            }
        }
        part.markingFeedback.filter(m=>m.message).forEach(m=>real_console.error(`* ${m.message}`));
        throw(new PartCreditError(`${niceName} got ${part.credit} credit`));
    }
}

async function run(numbasobject) {
    try {
        Numbas.debug = function() {};
        var data = parse_numbasobject(numbasobject);
        Numbas.custom_part_types = {};
        data.custom_part_types.forEach(d=>Numbas.custom_part_types[d.short_name] = d);
        var extension_names = data.extensions.map(p=>{
            var abspath = path.resolve(p);
            var location = path.basename(abspath);
            fs.readdirSync(abspath).forEach(f=>{
                if(path.extname(f)=='.js') {
                    const fname = path.resolve(abspath,f);
                    const obj = require(fname);
                }
            });
            return location;
        });
        var extensions = extension_names.map(location=>`extensions/${location}/${location}.js`);
        var reqs = standard_reqs.concat(extensions);
        Numbas.runImmediately(reqs,async function() {
            for(var name in Numbas.custom_part_types) {
                Numbas.partConstructors[name] = Numbas.parts.CustomPart;
            };
            extension_names.forEach(extension=>Numbas.activateExtension(extension));
            var scope = new Numbas.jme.Scope(Numbas.jme.builtinScope);
            for(var extension in Numbas.extensions) {
                if('scope' in Numbas.extensions[extension]) {
                    scope = new Numbas.jme.Scope([scope,Numbas.extensions[extension].scope]);
                }
            }
            var promises = [];
            data.question_groups.forEach(group=>{
                group.questions.forEach(q_data=>{
                    var q = Numbas.createQuestionFromJSON(q_data, 0, null, null, scope);
                    var promise = test_question(q);
                    promises.push(promise);
                });
            });
            try {
                await Promise.all(promises);
                succeed();
            } catch(e) {
                die(e);
            }
        });
    } catch(e) {
        die(e);
    }
}

if(process.stdin.isTTY) {
    real_console.error("You must pipe a question definition into this command.");
    process.exit(1);
}

var data = '';
process.stdin.setEncoding('utf-8');
process.stdin.on('readable', function() {
    var chunk;
    while (chunk = process.stdin.read()) {
        data += chunk;
    }
});

process.stdin.on('end', function () {
    run(data);
});