Skip to content

Commit f6d6ac9

Browse files
committed
get all tests working, various small bug fixes and updates
1 parent cfd6242 commit f6d6ac9

File tree

4 files changed

+68
-61
lines changed

4 files changed

+68
-61
lines changed

Makefile-mac

+3-20
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,13 @@ build-dev-etna-images:
1919
cd development-certs && docker build . -t development-certs
2020
cd docker/development-psql && docker build . -t development-psql
2121

22-
# Apache
22+
# # # Apache
2323
echo "Building apache images..."
2424
cd docker/etna-apache && docker build . -t etna-apache
2525
cd docker/etna-apache-v2 && docker build . -t etna-apache-v2
2626
cd edge-apache && docker build . -t edge-apache
2727

28-
# Etna base images
28+
# # Etna base images
2929
echo "Building etna images..."
3030
cd docker/etna-base-dev && docker build . -t etna-base-dev
3131
cd etna && docker build . -t etna
@@ -35,23 +35,6 @@ build-c4-dev-image:
3535
# Build c4-env image for Vulcan
3636
cd docker/vulcan_c4_env && docker build . -t vulcan_c4_env
3737

38-
build-dev-airflow-images:
39-
# Airflow
40-
echo "Building airflow images..."
41-
cd airflow-code-editor && docker build . -t airflow-code-editor
42-
cd etna/packages/etna-py && docker build . -t etna-py
43-
cd airflow && docker build . -t airflow
44-
45-
build-dev-archimedes-images:
46-
# Archimedes
47-
echo "Building archimedes images..."
48-
cd docker/archimedes-base && docker build . -t archimedes-base
49-
cd archimedes-r-base && docker build . -t archimedes-r-base # takes a really long time!
50-
cd docker/archimedes-node-base && docker build . -t archimedes-node-base
51-
cd archimedes && docker build . -t archimedes
52-
cd archimedes-r && docker build . -t archimedes-r
53-
cd archimedes-node && docker build . -t archimedes-node
54-
5538
### Project level targets ###
5639

5740
# Note: the project level docker compose files use relative paths so it is important to execute commands from within
@@ -88,7 +71,7 @@ etna-remove:
8871
cd etna && docker compose down
8972

9073
etna-libs-ruby:
91-
cd etna && docker compose run --rm -e FULL_BUILD=1 -e UPDATE_STATE=1 etna_app bundle install
74+
cd etna && docker compose run --rm -e FULL_BUILD=1 -e UPDATE_STATE=1 etna_app bundle install # This doesnt work anymore
9275

9376
etna-libs-js:
9477
cd etna && docker compose run --rm -e FULL_BUILD=1 -e UPDATE_STATE=1 etna_app bash -c 'cd packages/etna-js && npm install'

vulcan/lib/server/controllers/vulcan_v2_controller.rb

+4-5
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ def run_workflow
201201
end
202202

203203
# Build snakemake command
204-
available_files = @remote_manager.list_files(Vulcan::Path.workspace_output_path(workspace.path)).map { |file| "output/#{file}" },
204+
available_files = @remote_manager.list_files(Vulcan::Path.workspace_output_path(workspace.path)).map { |file| "output/#{file}" }
205205
params = @remote_manager.read_yaml_file(config.path).keys
206206
command = Vulcan::Snakemake::CommandBuilder.new
207207
command.targets = Vulcan::Snakemake::Inference.find_buildable_targets(workspace.target_mapping, params, available_files)
@@ -249,13 +249,12 @@ def get_workflow_status
249249
def write_files
250250
workspace = Vulcan::Workspace.first(id: @params[:workspace_id])
251251
raise Etna::BadRequest.new("Workspace not found") unless workspace
252-
files = @params || {}
252+
files = @params[:files] || {}
253253
raise Etna::BadRequest.new("No files provided") if files.empty?
254-
255254
begin
256-
files.each do |file_name, content|
255+
files.each do |file|
257256
output_path = Vulcan::Path.workspace_output_path(workspace.path)
258-
@remote_manager.write_file("#{output_path}#{file_name}", content)
257+
@remote_manager.write_file("#{output_path}#{file[:filename]}", file[:content])
259258
end
260259
rescue => e
261260
Vulcan.instance.logger.log_error(e)

vulcan/spec/spec_helper.rb

+8-3
Original file line numberDiff line numberDiff line change
@@ -322,8 +322,13 @@ def write_files_to_workspace(workspace_id)
322322
# The first step in the test workflow involves the UI writing files to the workspace
323323
auth_header(:editor)
324324
request = {
325-
"poem.txt" => poem_1_text,
326-
"poem_2.txt" => poem_2_text
325+
files: [{
326+
filename: "poem.txt",
327+
content: poem_1_text
328+
}, {
329+
filename: "poem_2.txt",
330+
content: poem_2_text
331+
}]
327332
}
328333
post("/api/v2/#{PROJECT}/workspace/#{workspace_id}/file/write", request)
329334
expect(last_response.status).to eq(200)
@@ -336,7 +341,7 @@ def remove_all_dirs
336341
end
337342

338343

339-
def check_jobs_status(job_names, max_attempts = 10, base_delay = 10)
344+
def check_jobs_status(job_names, max_attempts = 5, base_delay = 10)
340345
attempts = 0
341346

342347
loop do

vulcan/spec/workflow_v2_spec.rb

+53-33
Original file line numberDiff line numberDiff line change
@@ -466,7 +466,10 @@ def app
466466
file_name = "test_file_name"
467467
content = "This is a test file, with content 1."
468468
request = {
469-
file_name => content
469+
files: [{
470+
filename: file_name,
471+
content: content
472+
}]
470473
}
471474
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/file/write", request)
472475
expect(last_response.status).to eq(200)
@@ -499,7 +502,12 @@ def app
499502
it 'requests a file from the workspace' do
500503
# Write a file to the workspace
501504
workspace_id = Vulcan::Workspace.all[0].id
502-
request = {"test_file_name": "This is a test file, with content 1"}
505+
request = {
506+
files: [{
507+
filename: "test_file_name",
508+
content: "This is a test file, with content 1"
509+
}]
510+
}
503511
post("/api/v2/#{PROJECT}/workspace/#{workspace_id}/file/write", request)
504512
expect(last_response.status).to eq(200)
505513

@@ -558,6 +566,27 @@ def app
558566
expect(last_response.status).to eq(200)
559567
end
560568

569+
it 'alerts if snakemake is still running' do
570+
auth_header(:editor)
571+
workspace = Vulcan::Workspace.all[0]
572+
write_files_to_workspace(workspace.id)
573+
request = {
574+
params: {
575+
count_bytes: true,
576+
count_chars: false,
577+
add: 2,
578+
add_and_multiply_by: 4
579+
}
580+
}
581+
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/config", request)
582+
config_id = json_body[:config_id]
583+
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/run/#{config_id}")
584+
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/run/#{config_id}")
585+
expect(last_response.status).to eq(429)
586+
expect(json_body[:error]).to eq("workflow is still running...")
587+
end
588+
589+
561590
it 'can run the first step of a workflow' do
562591
auth_header(:editor)
563592
workspace = Vulcan::Workspace.all[0]
@@ -572,6 +601,7 @@ def app
572601
}
573602
}
574603
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/config", request)
604+
expect(last_response.status).to eq(200)
575605
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/run/#{json_body[:config_id]}")
576606
run_id = json_body[:run_id]
577607
expect(last_response.status).to eq(200)
@@ -609,7 +639,7 @@ def app
609639
expect(json_body[:run_id]).to_not be_nil
610640
run_id = json_body[:run_id]
611641
# Make sure jobs are finished
612-
check_jobs_status(["count", "arithmetic", "checker"]) do
642+
check_jobs_status(["count", "arithmetic", "checker"], 5) do
613643
get("/api/v2/#{PROJECT}/workspace/#{workspace.id}/run/#{run_id}")
614644
end
615645
# Outputs are created
@@ -624,7 +654,6 @@ def app
624654
expect(remote_manager.file_exists?(obj.log_path)).to be_truthy
625655
end
626656

627-
628657
it 'can run one step and then another' do
629658
auth_header(:editor)
630659
workspace = Vulcan::Workspace.all[0]
@@ -638,6 +667,7 @@ def app
638667
}
639668

640669
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/config", request)
670+
expect(last_response.status).to eq(200)
641671
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/run/#{json_body[:config_id]}")
642672
expect(last_response.status).to eq(200)
643673
run_id = json_body[:run_id]
@@ -659,7 +689,7 @@ def app
659689
expect(last_response.status).to eq(200)
660690
config_id = json_body[:config_id]
661691
# Sometimes snakemake still needs a minute to shut-down even though slurm reports the job as complete
662-
run_workflow_with_retry do
692+
run_workflow_with_retry(5) do
663693
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/run/#{config_id}")
664694
end
665695
expect(last_response.status).to eq(200)
@@ -700,18 +730,27 @@ def app
700730
}
701731
}
702732
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/config", request)
703-
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/run/#{json_body[:config_id]}")
733+
config_id = json_body[:config_id]
734+
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/run/#{config_id}")
704735
run_id = json_body[:run_id]
705736
check_jobs_status(["count", "arithmetic", "checker"]) do
706737
get("/api/v2/#{PROJECT}/workspace/#{workspace.id}/run/#{run_id}")
707738
end
708739
expect(last_response.status).to eq(200)
709740

710-
# Next step involves writing files to the workspace
711-
request = {"test_file_name": "This is a test file, with content 1"}
741+
# Next step involves UI steps - so we just simulate this by writing files to the workspace
742+
request = {
743+
files: [{
744+
filename: "ui_job_one.txt",
745+
content: "This is a test file, with content 1"
746+
}, {
747+
filename: "ui_job_two.txt",
748+
content: "This is a test file, with content 2"
749+
}]
750+
}
712751
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/file/write", request)
713752

714-
# Run the last job, send the same params as before
753+
# Run the last job, send the same params as before (this is what the UI does)
715754
request = {
716755
params: {
717756
count_bytes: true,
@@ -720,8 +759,9 @@ def app
720759
add_and_multiply_by: 4
721760
}
722761
}
762+
763+
# The config has not changed - just the files
723764
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/config", request)
724-
config_id = json_body[:config_id]
725765
run_workflow_with_retry do
726766
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/run/#{config_id}")
727767
end
@@ -751,26 +791,6 @@ def app
751791

752792
end
753793

754-
it 'alerts if snakemake is still running' do
755-
auth_header(:editor)
756-
workspace = Vulcan::Workspace.all[0]
757-
write_files_to_workspace(workspace.id)
758-
request = {
759-
params: {
760-
count_bytes: true,
761-
count_chars: false,
762-
add: 2,
763-
add_and_multiply_by: 4
764-
}
765-
}
766-
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/config", request)
767-
config_id = json_body[:config_id]
768-
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/run/#{config_id}")
769-
post("/api/v2/#{PROJECT}/workspace/#{workspace.id}/run/#{config_id}")
770-
expect(last_response.status).to eq(429)
771-
expect(json_body[:error]).to eq("workflow is still running...")
772-
end
773-
774794
end
775795

776796
context 'status checking', long_running: true do
@@ -790,14 +810,14 @@ def app
790810

791811
end
792812

793-
it 'invokes the first step of a workflow' do
813+
it 'checks the status of the first step of a workflow' do
794814
auth_header(:editor)
795815
workspace = Vulcan::Workspace.all[0]
796816
write_files_to_workspace(workspace.id)
797817
request = {
798818
params: {
799-
count_bytes: true,
800-
count_chars: false
819+
count_bytes: false,
820+
count_chars: true
801821
}
802822
}
803823
# TODO: add a meta key that can switch profiles

0 commit comments

Comments
 (0)