From aef1f221b883aa676b2dbeb0fee971b1904c5047 Mon Sep 17 00:00:00 2001 From: Tomoya Tanjo Date: Wed, 17 Feb 2021 12:53:53 +0000 Subject: [PATCH] Update requirements --- .devcontainer/Dockerfile | 2 +- .github/workflows/conformance.yml | 8 +-- lib/ep3/ep3-run.rb | 6 +- lib/ep3/init/cwl2wfnet.rb | 98 ++++++++++++++++++++++++++++++- lib/ep3/init/wfnet2medal.rb | 6 ++ lib/ep3/init/workflownet.rb | 16 +---- 6 files changed, 114 insertions(+), 22 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 5f9fde1..1a05dc8 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.12 +FROM alpine:3.13 RUN apk --no-cache add ruby ruby-json ruby-etc nodejs jq docker-cli \ ruby-irb ruby-webrick \ diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index d1cc75e..04a7c84 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -43,14 +43,14 @@ jobs: curl -SL https://github.com/tom-tan/medal/releases/download/${medal_ver}/medal-linux-x86_64.tar.gz \ | sudo tar xC /usr/bin env: - medal_ver: v0.0.11 - - uses: actions/setup-node@v1 + medal_ver: v0.4.0 + - uses: actions/setup-node@v2 with: - node-version: '12.x' + node-version: '14.x' - name: Setup Python for testing uses: actions/setup-python@v2 with: - python-version: '3.8.x' + python-version: '3.9.x' - name: Install cwltest run: pip install cwltest - name: Run example diff --git a/lib/ep3/ep3-run.rb b/lib/ep3/ep3-run.rb index 61f7986..afebf7e 100755 --- a/lib/ep3/ep3-run.rb +++ b/lib/ep3/ep3-run.rb @@ -81,7 +81,11 @@ def ep3_run(args) else '/dev/null' end - medal_pid = spawn({ 'PATH' => "#{ENV['EP3_LIBPATH']}/runtime:#{ENV['PATH']}" }, + env = { 'EP3_LIBPATH' => ENV['EP3_LIBPATH'], 'DOCKER_HOST' => ENV.fetch('DOCKER_HOST', '') } + if ENV.include? 'DOCKER_HOST' + env['DOCKER_HOST'] = ENV['DOCKER_HOST'] + end + medal_pid = spawn(env, "bash", "-o", "pipefail", "-c", "medal workdir/job.yml -i workdir/init.yml --workdir=workdir --tmpdir=tmpdir --leave-tmpdir --debug 3>&2 2>&1 1>&3 | tee #{logfile}", :chdir => dir, :err => :out, :out => debugout) diff --git a/lib/ep3/init/cwl2wfnet.rb b/lib/ep3/init/cwl2wfnet.rb index 195ead0..40ccd1b 100755 --- a/lib/ep3/init/cwl2wfnet.rb +++ b/lib/ep3/init/cwl2wfnet.rb @@ -81,13 +81,20 @@ def prepare(basefile, cfile, dst) def convert(dst) cwl = CommonWorkflowLanguage.load_file(File.join(dst, 'job.cwl'), false) case walk(cwl, '.class') - when 'CommandLineTool', 'ExpressionTool' + when 'CommandLineTool' [ { destination: File.join(*dst), net: cmdnet(cwl), } ] + when 'ExpressionTool' + [ + { + destination: File.join(*dst), + net: expnet(cwl), + } + ] when 'Workflow' net = { destination: File.join(*dst), @@ -102,7 +109,7 @@ def convert(dst) def cmdnet(cwl) any = '_' - net = PetriNet.new('command-line-tool', 'ep3.system.main') + net = PetriNet.new('command-line-tool', 'ep3.system.main', 'tool') net << Transition.new(in_: [Place.new('entrypoint', any)], out: [Place.new('input.json', "~(entrypoint)"), @@ -184,6 +191,91 @@ def cmdnet(cwl) net end +def expnet(cwl) + any = '_' + net = PetriNet.new('command-line-tool', 'ep3.system.main', 'expression') + + net << Transition.new(in_: [Place.new('entrypoint', any)], + out: [Place.new('input.json', "~(entrypoint)"), + Place.new('StageIn', 'not-started'), Place.new('CommandGeneration', 'not-started'), + Place.new('Execution', 'not-started'), Place.new('StageOut', 'not-started')], + name: 'prepare') + net << Transition.new(in_: [Place.new('StageIn', 'not-started'), Place.new('input.json', any)], + out: [Place.new('StageIn', 'success'), + Place.new('cwl.input.json', 'STDOUT'), Place.new('StageIn.err', 'STDERR')], + command: %q!mkdir -p $MEDAL_TMPDIR/outputs; stage-in.rb --outdir=$MEDAL_TMPDIR/outputs job.cwl ~(input.json)!, + name: 'stage-in') + + net << Transition.new(in_: [Place.new('CommandGeneration', 'not-started'), Place.new('StageIn', 'success'), + Place.new('cwl.input.json', any)], + out: [Place.new('CommandGeneration', 'success'), Place.new('cwl.input.json', '~(cwl.input.json)'), + Place.new('CommandGeneration.command', 'STDOUT'), + Place.new('CommandGeneration.err', 'STDERR')], + command: %Q!inspector.rb job.cwl commandline -i ~(cwl.input.json) --outdir=$MEDAL_TMPDIR/outputs!, + name: 'generate-command') + + net << Transition.new(in_: [Place.new('Execution', 'not-started'), Place.new('CommandGeneration', 'success'), + Place.new('CommandGeneration.command', any)], + out: [Place.new('Execution.return', 'RETURN'), Place.new('Execution.out', 'STDOUT'), Place.new('Execution.err', 'STDERR')], + command: %Q!executor ~(CommandGeneration.command)!, + name: 'execute') + + successCodes = case cwl.class_ + when 'CommandLineTool' + cwl.successCodes + when 'ExpressionTool' + [0] + end + successCodes.each{ |c| + net << Transition.new(in_: [Place.new('Execution.return', c.to_s)], + out: [Place.new('Execution', 'success')], + name: 'verify-success') + } + + temporaryFailCodes = case cwl.class_ + when 'CommandLineTool' + cwl.temporaryFailCodes + when 'ExpressionTool' + [] + end + temporaryFailCodes.each{ |c| + net << Transition.new(in_: [Place.new('Execution.return', c.to_s)], + out: [Place.new('Execution', 'temporaryFailure')], + name: 'verify-temporaryFailure') + } + unless temporaryFailCodes.empty? + net << Transition.new(in_: [Place.new('Execution', 'temporaryFailure')], out: [], + command: '"false"', + name: 'fail') + end + + permanentFailCodes = case cwl.class_ + when 'CommandLineTool' + codes = cwl.permanentFailCodes + codes.empty? ? [any] : codes + when 'ExpressionTool' + [any] + end + permanentFailCodes.each{ |c| + net << Transition.new(in_: [Place.new('Execution.return', c.to_s)], out: [Place.new('Execution', 'permanentFailure')], + name: 'verify-permanentFailure') + } + unless permanentFailCodes.empty? + net << Transition.new(in_: [Place.new('Execution', 'permanentFailure')], out: [], + command: '"false"', + name: 'fail') + end + + net << Transition.new(in_: [Place.new('StageOut', 'not-started'), Place.new('Execution', 'success'), + Place.new('cwl.input.json', any)], + out: [Place.new('StageOut', 'success'), Place.new('ExecutionState', 'success'), + Place.new('cwl.output.json', 'STDOUT'), Place.new('StageOut.err', 'STDERR')], + command: %Q!inspector.rb job.cwl list -i ~(cwl.input.json) --json --outdir=$MEDAL_TMPDIR/outputs!, + name: 'stage-out') + net +end + + def default_inputs_for_steps(cwl) ret = {} cwl.steps.each{ |s| @@ -243,7 +335,7 @@ def default_inputs_for_steps(cwl) def wfnet(cwl) any = '_' - net = PetriNet.new('workflow', 'ep3.system.main') + net = PetriNet.new('workflow', 'ep3.system.main', 'workflow') # prevStep: [nextStep] outConnections = Hash.new{ |hash, key| hash[key] = [] } diff --git a/lib/ep3/init/wfnet2medal.rb b/lib/ep3/init/wfnet2medal.rb index f25156a..859dfa6 100644 --- a/lib/ep3/init/wfnet2medal.rb +++ b/lib/ep3/init/wfnet2medal.rb @@ -63,7 +63,13 @@ def wfnet2medal(net) } <