diff --git a/example/docker-compose.yml b/example/docker-compose.yml index 04957e3..f6fa763 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -5,7 +5,7 @@ services: - -m - pirlib.backends.docker_batch - graph - - gASV+wAAAAAAAABdlCiMDHBpcmxpYi5ncmFwaJSMC0dyYXBoT3V0cHV0lJOUKYGUfZQojARuYW1llIwIcmV0dXJuLjCUjAZpb3R5cGWUjARGSUxFlIwGc291cmNllGgBjApEYXRhU291cmNllJOUKYGUfZQojARub2RllIwFdHJhaW6UjAhzdWJncmFwaJROjAZvdXRwdXSUjAZyZXR1cm6UjAtncmFwaF9pbnB1dJROdWJ1YmgDKYGUfZQoaAaMCHJldHVybi4xlGgIjAlEQVRBRlJBTUWUaApoDCmBlH2UKGgPjAhldmFsdWF0ZZRoEU5oEowGcmV0dXJulGgUTnVidWJlLg== + - gASV+QAAAAAAAABdlCiMCnBpcmxpYi5waXKUjAtHcmFwaE91dHB1dJSTlCmBlH2UKIwEbmFtZZSMCHJldHVybi4wlIwGaW90eXBllIwERklMRZSMBnNvdXJjZZRoAYwKRGF0YVNvdXJjZZSTlCmBlH2UKIwEbm9kZZSMBXRyYWlulIwIc3ViZ3JhcGiUTowGb3V0cHV0lIwGcmV0dXJulIwLZ3JhcGhfaW5wdXSUTnVidWJoAymBlH2UKGgGjAhyZXR1cm4uMZRoCIwJREFUQUZSQU1FlGgKaAwpgZR9lChoD4wIZXZhbHVhdGWUaBFOaBKMBnJldHVybpRoFE51YnViZS4= depends_on: train_pipeline.clean: condition: service_completed_successfully @@ -17,7 +17,7 @@ services: condition: service_completed_successfully train_pipeline.train: condition: service_completed_successfully - image: pircli-build:6063df32-2483-4e2e-beee-944931c72937 + image: pircli-build:fbb9b1d1-11ed-48a6-a70c-67a1f6c928a5 volumes: - node_outputs:/mnt/node_outputs - ${INPUT_train_dataset:?err}:/mnt/graph_inputs/train_dataset @@ -30,9 +30,9 @@ services: - -m - pirlib.backends.docker_batch - node - - gASV1wEAAAAAAACMDHBpcmxpYi5ncmFwaJSMBE5vZGWUk5QpgZR9lCiMBG5hbWWUjAVjbGVhbpSMBmNvbmZpZ5R9lIwGaW5wdXRzlF2UaACMBUlucHV0lJOUKYGUfZQoaAWMB2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjAZzb3VyY2WUaACMCkRhdGFTb3VyY2WUk5QpgZR9lCiMBG5vZGWUTowIc3ViZ3JhcGiUTowGb3V0cHV0lE6MC2dyYXBoX2lucHV0lIwNdHJhaW5fZGF0YXNldJR1YnViYYwHb3V0cHV0c5RdlGgAjAZPdXRwdXSUk5QpgZR9lChoBYwGcmV0dXJulGgQjAlESVJFQ1RPUlmUdWJhjAplbnRyeXBvaW50lGgAjApFbnRyeXBvaW50lJOUKYGUfZQojAd2ZXJzaW9ulIwCdjGUjAdoYW5kbGVylIwVZXhhbXBsZS5leGFtcGxlOmNsZWFulIwHcnVudGltZZSMCnB5dGhvbjozLjiUjAdjb2RldXJplE6MBWltYWdllIwxcGlyY2xpLWJ1aWxkOjYwNjNkZjMyLTI0ODMtNGUyZS1iZWVlLTk0NDkzMWM3MjkzN5R1YowJZnJhbWV3b3JrlE51Yi4= - - gASVpAAAAAAAAABdlCiMDHBpcmxpYi5ncmFwaJSMCkdyYXBoSW5wdXSUk5QpgZR9lCiMBG5hbWWUjA10cmFpbl9kYXRhc2V0lIwGaW90eXBllIwJRElSRUNUT1JZlHViaAMpgZR9lChoBowPdHJhbnNsYXRlX21vZGVslGgIjARGSUxFlHViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUdWJlLg== - image: pircli-build:6063df32-2483-4e2e-beee-944931c72937 + - gASV1QEAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojARuYW1llIwFY2xlYW6UjAplbnRyeXBvaW50lGgAjApFbnRyeXBvaW50lJOUKYGUfZQojAd2ZXJzaW9ulIwCdjGUjAdoYW5kbGVylIwVZXhhbXBsZS5leGFtcGxlOmNsZWFulIwHcnVudGltZZSMCnB5dGhvbjozLjiUjAdjb2RldXJslE6MBWltYWdllIwxcGlyY2xpLWJ1aWxkOmZiYjliMWQxLTExZWQtNDhhNi1hNzBjLTY3YTFmNmM5MjhhNZR1YowJZnJhbWV3b3JrlE6MBmNvbmZpZ5R9lIwGaW5wdXRzlF2UaACMBUlucHV0lJOUKYGUfZQoaAWMB2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjAZzb3VyY2WUaACMCkRhdGFTb3VyY2WUk5QpgZR9lCiMBG5vZGWUTowIc3ViZ3JhcGiUTowGb3V0cHV0lE6MC2dyYXBoX2lucHV0lIwNdHJhaW5fZGF0YXNldJR1YnViYYwHb3V0cHV0c5RdlGgAjAZPdXRwdXSUk5QpgZR9lChoBYwGcmV0dXJulGgfjAlESVJFQ1RPUlmUdWJhdWIu + - gASVogAAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojARuYW1llIwNdHJhaW5fZGF0YXNldJSMBmlvdHlwZZSMCURJUkVDVE9SWZR1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZR1YmgDKYGUfZQoaAaMCXNlbnRlbmNlc5RoCIwJRElSRUNUT1JZlHViZS4= + image: pircli-build:fbb9b1d1-11ed-48a6-a70c-67a1f6c928a5 volumes: - node_outputs:/mnt/node_outputs - ${INPUT_train_dataset:?err}:/mnt/graph_inputs/train_dataset @@ -42,12 +42,12 @@ services: - -m - pirlib.backends.docker_batch - node - - gASVTwIAAAAAAACMDHBpcmxpYi5ncmFwaJSMBE5vZGWUk5QpgZR9lCiMBG5hbWWUjAhldmFsdWF0ZZSMBmNvbmZpZ5R9lIwGaW5wdXRzlF2UKGgAjAVJbnB1dJSTlCmBlH2UKGgFjBNrd2FyZ3MudGVzdF9kYXRhc2V0lIwGaW90eXBllIwJRElSRUNUT1JZlIwGc291cmNllGgAjApEYXRhU291cmNllJOUKYGUfZQojARub2RllE6MCHN1YmdyYXBolE6MBm91dHB1dJROjAtncmFwaF9pbnB1dJSMCXNlbnRlbmNlc5R1YnViaAwpgZR9lChoBYwSa3dhcmdzLnByZWRpY3Rpb25zlGgQjAlESVJFQ1RPUlmUaBJoFCmBlH2UKGgXjBhpbmZlcl9waXBlbGluZS5zZW50aW1lbnSUaBhOaBmMBnJldHVybpRoGk51YnViZYwHb3V0cHV0c5RdlGgAjAZPdXRwdXSUk5QpgZR9lChoBYwGcmV0dXJulGgQjAlEQVRBRlJBTUWUdWJhjAplbnRyeXBvaW50lGgAjApFbnRyeXBvaW50lJOUKYGUfZQojAd2ZXJzaW9ulIwCdjGUjAdoYW5kbGVylIwYZXhhbXBsZS5leGFtcGxlOmV2YWx1YXRllIwHcnVudGltZZSMCnB5dGhvbjozLjiUjAdjb2RldXJplE6MBWltYWdllIwxcGlyY2xpLWJ1aWxkOjYwNjNkZjMyLTI0ODMtNGUyZS1iZWVlLTk0NDkzMWM3MjkzN5R1YowJZnJhbWV3b3JrlE51Yi4= - - gASVpAAAAAAAAABdlCiMDHBpcmxpYi5ncmFwaJSMCkdyYXBoSW5wdXSUk5QpgZR9lCiMBG5hbWWUjA10cmFpbl9kYXRhc2V0lIwGaW90eXBllIwJRElSRUNUT1JZlHViaAMpgZR9lChoBowPdHJhbnNsYXRlX21vZGVslGgIjARGSUxFlHViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUdWJlLg== + - gASVTQIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojARuYW1llIwIZXZhbHVhdGWUjAplbnRyeXBvaW50lGgAjApFbnRyeXBvaW50lJOUKYGUfZQojAd2ZXJzaW9ulIwCdjGUjAdoYW5kbGVylIwYZXhhbXBsZS5leGFtcGxlOmV2YWx1YXRllIwHcnVudGltZZSMCnB5dGhvbjozLjiUjAdjb2RldXJslE6MBWltYWdllIwxcGlyY2xpLWJ1aWxkOmZiYjliMWQxLTExZWQtNDhhNi1hNzBjLTY3YTFmNmM5MjhhNZR1YowJZnJhbWV3b3JrlE6MBmNvbmZpZ5R9lIwGaW5wdXRzlF2UKGgAjAVJbnB1dJSTlCmBlH2UKGgFjBNrd2FyZ3MudGVzdF9kYXRhc2V0lIwGaW90eXBllIwJRElSRUNUT1JZlIwGc291cmNllGgAjApEYXRhU291cmNllJOUKYGUfZQojARub2RllE6MCHN1YmdyYXBolE6MBm91dHB1dJROjAtncmFwaF9pbnB1dJSMCXNlbnRlbmNlc5R1YnViaBspgZR9lChoBYwSa3dhcmdzLnByZWRpY3Rpb25zlGgfjAlESVJFQ1RPUlmUaCFoIymBlH2UKGgmjBhpbmZlcl9waXBlbGluZS5zZW50aW1lbnSUaCdOaCiMBnJldHVybpRoKU51YnViZYwHb3V0cHV0c5RdlGgAjAZPdXRwdXSUk5QpgZR9lChoBYwGcmV0dXJulGgfjAlEQVRBRlJBTUWUdWJhdWIu + - gASVogAAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojARuYW1llIwNdHJhaW5fZGF0YXNldJSMBmlvdHlwZZSMCURJUkVDVE9SWZR1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZR1YmgDKYGUfZQoaAaMCXNlbnRlbmNlc5RoCIwJRElSRUNUT1JZlHViZS4= depends_on: train_pipeline.infer_pipeline.sentiment: condition: service_completed_successfully - image: pircli-build:6063df32-2483-4e2e-beee-944931c72937 + image: pircli-build:fbb9b1d1-11ed-48a6-a70c-67a1f6c928a5 volumes: - node_outputs:/mnt/node_outputs - ${INPUT_sentences:?err}:/mnt/graph_inputs/sentences @@ -57,14 +57,14 @@ services: - -m - pirlib.backends.docker_batch - node - - gASVSgIAAAAAAACMDHBpcmxpYi5ncmFwaJSMBE5vZGWUk5QpgZR9lCiMBG5hbWWUjBhpbmZlcl9waXBlbGluZS5zZW50aW1lbnSUjAZjb25maWeUfZSMBmlucHV0c5RdlChoAIwFSW5wdXSUk5QpgZR9lChoBYwFbW9kZWyUjAZpb3R5cGWUjARGSUxFlIwGc291cmNllGgAjApEYXRhU291cmNllJOUKYGUfZQojARub2RllIwFdHJhaW6UjAhzdWJncmFwaJROjAZvdXRwdXSUjAZyZXR1cm6UjAtncmFwaF9pbnB1dJROdWJ1YmgMKYGUfZQoaAWMCXNlbnRlbmNlc5RoEIwJRElSRUNUT1JZlGgSaBQpgZR9lChoF4waaW5mZXJfcGlwZWxpbmUudHJhbnNsYXRlXzGUaBlOaBqMBnJldHVybpRoHE51YnViZYwHb3V0cHV0c5RdlGgAjAZPdXRwdXSUk5QpgZR9lChoBYwGcmV0dXJulGgQjAlESVJFQ1RPUlmUdWJhjAplbnRyeXBvaW50lGgAjApFbnRyeXBvaW50lJOUKYGUfZQojAd2ZXJzaW9ulIwCdjGUjAdoYW5kbGVylIwZZXhhbXBsZS5leGFtcGxlOnNlbnRpbWVudJSMB3J1bnRpbWWUjApweXRob246My44lIwHY29kZXVyaZROjAVpbWFnZZSMMXBpcmNsaS1idWlsZDo2MDYzZGYzMi0yNDgzLTRlMmUtYmVlZS05NDQ5MzFjNzI5MzeUdWKMCWZyYW1ld29ya5ROdWIu - - gASVpAAAAAAAAABdlCiMDHBpcmxpYi5ncmFwaJSMCkdyYXBoSW5wdXSUk5QpgZR9lCiMBG5hbWWUjA10cmFpbl9kYXRhc2V0lIwGaW90eXBllIwJRElSRUNUT1JZlHViaAMpgZR9lChoBowPdHJhbnNsYXRlX21vZGVslGgIjARGSUxFlHViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUdWJlLg== + - gASVSAIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojARuYW1llIwYaW5mZXJfcGlwZWxpbmUuc2VudGltZW50lIwKZW50cnlwb2ludJRoAIwKRW50cnlwb2ludJSTlCmBlH2UKIwHdmVyc2lvbpSMAnYxlIwHaGFuZGxlcpSMGWV4YW1wbGUuZXhhbXBsZTpzZW50aW1lbnSUjAdydW50aW1llIwKcHl0aG9uOjMuOJSMB2NvZGV1cmyUTowFaW1hZ2WUjDFwaXJjbGktYnVpbGQ6ZmJiOWIxZDEtMTFlZC00OGE2LWE3MGMtNjdhMWY2YzkyOGE1lHVijAlmcmFtZXdvcmuUTowGY29uZmlnlH2UjAZpbnB1dHOUXZQoaACMBUlucHV0lJOUKYGUfZQoaAWMBW1vZGVslIwGaW90eXBllIwERklMRZSMBnNvdXJjZZRoAIwKRGF0YVNvdXJjZZSTlCmBlH2UKIwEbm9kZZSMBXRyYWlulIwIc3ViZ3JhcGiUTowGb3V0cHV0lIwGcmV0dXJulIwLZ3JhcGhfaW5wdXSUTnVidWJoGymBlH2UKGgFjAlzZW50ZW5jZXOUaB+MCURJUkVDVE9SWZRoIWgjKYGUfZQoaCaMGmluZmVyX3BpcGVsaW5lLnRyYW5zbGF0ZV8xlGgoTmgpjAZyZXR1cm6UaCtOdWJ1YmWMB291dHB1dHOUXZRoAIwGT3V0cHV0lJOUKYGUfZQoaAWMBnJldHVybpRoH4wJRElSRUNUT1JZlHViYXViLg== + - gASVogAAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojARuYW1llIwNdHJhaW5fZGF0YXNldJSMBmlvdHlwZZSMCURJUkVDVE9SWZR1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZR1YmgDKYGUfZQoaAaMCXNlbnRlbmNlc5RoCIwJRElSRUNUT1JZlHViZS4= depends_on: train_pipeline.infer_pipeline.translate_1: condition: service_completed_successfully train_pipeline.train: condition: service_completed_successfully - image: pircli-build:6063df32-2483-4e2e-beee-944931c72937 + image: pircli-build:fbb9b1d1-11ed-48a6-a70c-67a1f6c928a5 volumes: - node_outputs:/mnt/node_outputs train_pipeline.infer_pipeline.translate_1: @@ -73,9 +73,9 @@ services: - -m - pirlib.backends.docker_batch - node - - gASVQgIAAAAAAACMDHBpcmxpYi5ncmFwaJSMBE5vZGWUk5QpgZR9lCiMBG5hbWWUjBppbmZlcl9waXBlbGluZS50cmFuc2xhdGVfMZSMBmNvbmZpZ5R9lIwDa2V5lIwFdmFsdWWUc4wGaW5wdXRzlF2UKGgAjAVJbnB1dJSTlCmBlH2UKGgFjAZhcmdzLjCUjAZpb3R5cGWUjARGSUxFlIwGc291cmNllGgAjApEYXRhU291cmNllJOUKYGUfZQojARub2RllE6MCHN1YmdyYXBolE6MBm91dHB1dJROjAtncmFwaF9pbnB1dJSMD3RyYW5zbGF0ZV9tb2RlbJR1YnViaA4pgZR9lChoBYwGYXJncy4xlGgSjAlESVJFQ1RPUlmUaBRoFimBlH2UKGgZTmgaTmgbTmgcjAlzZW50ZW5jZXOUdWJ1YmWMB291dHB1dHOUXZRoAIwGT3V0cHV0lJOUKYGUfZQoaAWMBnJldHVybpRoEowJRElSRUNUT1JZlHViYYwKZW50cnlwb2ludJRoAIwKRW50cnlwb2ludJSTlCmBlH2UKIwHdmVyc2lvbpSMAnYxlIwHaGFuZGxlcpSMGWV4YW1wbGUuZXhhbXBsZTp0cmFuc2xhdGWUjAdydW50aW1llIwKcHl0aG9uOjMuOJSMB2NvZGV1cmmUTowFaW1hZ2WUjDFwaXJjbGktYnVpbGQ6NjA2M2RmMzItMjQ4My00ZTJlLWJlZWUtOTQ0OTMxYzcyOTM3lHVijAlmcmFtZXdvcmuUTnViLg== - - gASVpAAAAAAAAABdlCiMDHBpcmxpYi5ncmFwaJSMCkdyYXBoSW5wdXSUk5QpgZR9lCiMBG5hbWWUjA10cmFpbl9kYXRhc2V0lIwGaW90eXBllIwJRElSRUNUT1JZlHViaAMpgZR9lChoBowPdHJhbnNsYXRlX21vZGVslGgIjARGSUxFlHViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUdWJlLg== - image: pircli-build:6063df32-2483-4e2e-beee-944931c72937 + - gASVQAIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojARuYW1llIwaaW5mZXJfcGlwZWxpbmUudHJhbnNsYXRlXzGUjAplbnRyeXBvaW50lGgAjApFbnRyeXBvaW50lJOUKYGUfZQojAd2ZXJzaW9ulIwCdjGUjAdoYW5kbGVylIwZZXhhbXBsZS5leGFtcGxlOnRyYW5zbGF0ZZSMB3J1bnRpbWWUjApweXRob246My44lIwHY29kZXVybJROjAVpbWFnZZSMMXBpcmNsaS1idWlsZDpmYmI5YjFkMS0xMWVkLTQ4YTYtYTcwYy02N2ExZjZjOTI4YTWUdWKMCWZyYW1ld29ya5ROjAZjb25maWeUfZSMA2tleZSMBXZhbHVllHOMBmlucHV0c5RdlChoAIwFSW5wdXSUk5QpgZR9lChoBYwGYXJncy4wlIwGaW90eXBllIwERklMRZSMBnNvdXJjZZRoAIwKRGF0YVNvdXJjZZSTlCmBlH2UKIwEbm9kZZROjAhzdWJncmFwaJROjAZvdXRwdXSUTowLZ3JhcGhfaW5wdXSUjA90cmFuc2xhdGVfbW9kZWyUdWJ1YmgdKYGUfZQoaAWMBmFyZ3MuMZRoIYwJRElSRUNUT1JZlGgjaCUpgZR9lChoKE5oKU5oKk5oK4wJc2VudGVuY2VzlHVidWJljAdvdXRwdXRzlF2UaACMBk91dHB1dJSTlCmBlH2UKGgFjAZyZXR1cm6UaCGMCURJUkVDVE9SWZR1YmF1Yi4= + - gASVogAAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojARuYW1llIwNdHJhaW5fZGF0YXNldJSMBmlvdHlwZZSMCURJUkVDVE9SWZR1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZR1YmgDKYGUfZQoaAaMCXNlbnRlbmNlc5RoCIwJRElSRUNUT1JZlHViZS4= + image: pircli-build:fbb9b1d1-11ed-48a6-a70c-67a1f6c928a5 volumes: - node_outputs:/mnt/node_outputs - ${INPUT_translate_model:?err}:/mnt/graph_inputs/translate_model @@ -86,12 +86,12 @@ services: - -m - pirlib.backends.docker_batch - node - - gASVHQIAAAAAAACMDHBpcmxpYi5ncmFwaJSMBE5vZGWUk5QpgZR9lCiMBG5hbWWUjAV0cmFpbpSMBmNvbmZpZ5R9lIwGaW5wdXRzlF2UaACMBUlucHV0lJOUKYGUfZQoaAWMB2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjAZzb3VyY2WUaACMCkRhdGFTb3VyY2WUk5QpgZR9lCiMBG5vZGWUjAVjbGVhbpSMCHN1YmdyYXBolE6MBm91dHB1dJSMBnJldHVybpSMC2dyYXBoX2lucHV0lE51YnViYYwHb3V0cHV0c5RdlGgAjAZPdXRwdXSUk5QpgZR9lChoBYwGcmV0dXJulGgQjARGSUxFlHViYYwKZW50cnlwb2ludJRoAIwKRW50cnlwb2ludJSTlCmBlH2UKIwHdmVyc2lvbpSMAnYxlIwHaGFuZGxlcpSMFWV4YW1wbGUuZXhhbXBsZTp0cmFpbpSMB3J1bnRpbWWUjApweXRob246My44lIwHY29kZXVyaZROjAVpbWFnZZSMMXBpcmNsaS1idWlsZDo2MDYzZGYzMi0yNDgzLTRlMmUtYmVlZS05NDQ5MzFjNzI5MzeUdWKMCWZyYW1ld29ya5RoAIwJRnJhbWV3b3JrlJOUKYGUfZQoaAWMB2FkYXB0ZGyUaAd9lCiMDG1pbl9yZXBsaWNhc5RLAYwMbWF4X3JlcGxpY2FzlEsEdXVidWIu - - gASVpAAAAAAAAABdlCiMDHBpcmxpYi5ncmFwaJSMCkdyYXBoSW5wdXSUk5QpgZR9lCiMBG5hbWWUjA10cmFpbl9kYXRhc2V0lIwGaW90eXBllIwJRElSRUNUT1JZlHViaAMpgZR9lChoBowPdHJhbnNsYXRlX21vZGVslGgIjARGSUxFlHViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUdWJlLg== + - gASVGwIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojARuYW1llIwFdHJhaW6UjAplbnRyeXBvaW50lGgAjApFbnRyeXBvaW50lJOUKYGUfZQojAd2ZXJzaW9ulIwCdjGUjAdoYW5kbGVylIwVZXhhbXBsZS5leGFtcGxlOnRyYWlulIwHcnVudGltZZSMCnB5dGhvbjozLjiUjAdjb2RldXJslE6MBWltYWdllIwxcGlyY2xpLWJ1aWxkOmZiYjliMWQxLTExZWQtNDhhNi1hNzBjLTY3YTFmNmM5MjhhNZR1YowJZnJhbWV3b3JrlGgAjAlGcmFtZXdvcmuUk5QpgZR9lChoBYwHYWRhcHRkbJSMBmNvbmZpZ5R9lCiMDG1pbl9yZXBsaWNhc5RLAYwMbWF4X3JlcGxpY2FzlEsEdXViaBt9lIwGaW5wdXRzlF2UaACMBUlucHV0lJOUKYGUfZQoaAWMB2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjAZzb3VyY2WUaACMCkRhdGFTb3VyY2WUk5QpgZR9lCiMBG5vZGWUjAVjbGVhbpSMCHN1YmdyYXBolE6MBm91dHB1dJSMBnJldHVybpSMC2dyYXBoX2lucHV0lE51YnViYYwHb3V0cHV0c5RdlGgAjAZPdXRwdXSUk5QpgZR9lChoBYwGcmV0dXJulGgnjARGSUxFlHViYXViLg== + - gASVogAAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojARuYW1llIwNdHJhaW5fZGF0YXNldJSMBmlvdHlwZZSMCURJUkVDVE9SWZR1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZR1YmgDKYGUfZQoaAaMCXNlbnRlbmNlc5RoCIwJRElSRUNUT1JZlHViZS4= depends_on: train_pipeline.clean: condition: service_completed_successfully - image: pircli-build:6063df32-2483-4e2e-beee-944931c72937 + image: pircli-build:fbb9b1d1-11ed-48a6-a70c-67a1f6c928a5 volumes: - node_outputs:/mnt/node_outputs version: '3.9' diff --git a/example/package_docker.yml b/example/package_docker.yml index 5b697f0..86eec4a 100644 --- a/example/package_docker.yml +++ b/example/package_docker.yml @@ -2,6 +2,13 @@ graphs: - name: train_pipeline nodes: - name: clean + entrypoint: + version: v1 + handler: example.example:clean + runtime: python:3.8 + codeurl: null + image: pircli-build:fbb9b1d1-11ed-48a6-a70c-67a1f6c928a5 + framework: null config: {} inputs: - name: dataset @@ -14,14 +21,18 @@ graphs: outputs: - name: return iotype: DIRECTORY + - name: train entrypoint: version: v1 - handler: example.example:clean + handler: example.example:train runtime: python:3.8 - codeuri: null - image: pircli-build:6063df32-2483-4e2e-beee-944931c72937 - framework: null - - name: train + codeurl: null + image: pircli-build:fbb9b1d1-11ed-48a6-a70c-67a1f6c928a5 + framework: + name: adaptdl + config: + min_replicas: 1 + max_replicas: 4 config: {} inputs: - name: dataset @@ -34,18 +45,14 @@ graphs: outputs: - name: return iotype: FILE + - name: evaluate entrypoint: version: v1 - handler: example.example:train + handler: example.example:evaluate runtime: python:3.8 - codeuri: null - image: pircli-build:6063df32-2483-4e2e-beee-944931c72937 - framework: - name: adaptdl - config: - min_replicas: 1 - max_replicas: 4 - - name: evaluate + codeurl: null + image: pircli-build:fbb9b1d1-11ed-48a6-a70c-67a1f6c928a5 + framework: null config: {} inputs: - name: kwargs.test_dataset @@ -65,14 +72,14 @@ graphs: outputs: - name: return iotype: DATAFRAME + - name: infer_pipeline.translate_1 entrypoint: version: v1 - handler: example.example:evaluate + handler: example.example:translate runtime: python:3.8 - codeuri: null - image: pircli-build:6063df32-2483-4e2e-beee-944931c72937 + codeurl: null + image: pircli-build:fbb9b1d1-11ed-48a6-a70c-67a1f6c928a5 framework: null - - name: infer_pipeline.translate_1 config: key: value inputs: @@ -93,14 +100,14 @@ graphs: outputs: - name: return iotype: DIRECTORY + - name: infer_pipeline.sentiment entrypoint: version: v1 - handler: example.example:translate + handler: example.example:sentiment runtime: python:3.8 - codeuri: null - image: pircli-build:6063df32-2483-4e2e-beee-944931c72937 + codeurl: null + image: pircli-build:fbb9b1d1-11ed-48a6-a70c-67a1f6c928a5 framework: null - - name: infer_pipeline.sentiment config: {} inputs: - name: model @@ -120,13 +127,6 @@ graphs: outputs: - name: return iotype: DIRECTORY - entrypoint: - version: v1 - handler: example.example:sentiment - runtime: python:3.8 - codeuri: null - image: pircli-build:6063df32-2483-4e2e-beee-944931c72937 - framework: null subgraphs: [] inputs: - name: train_dataset diff --git a/example/package_inproc.yml b/example/package_inproc.yml index 594deb3..c0c54e4 100644 --- a/example/package_inproc.yml +++ b/example/package_inproc.yml @@ -6,7 +6,7 @@ graphs: version: v1 handler: example.example:translate runtime: python:3.8 - codeuri: null + codeurl: null image: null framework: null config: @@ -34,7 +34,7 @@ graphs: version: v1 handler: example.example:sentiment runtime: python:3.8 - codeuri: null + codeurl: null image: null framework: null config: {} @@ -79,7 +79,7 @@ graphs: version: v1 handler: example.example:clean runtime: python:3.8 - codeuri: null + codeurl: null image: null framework: null config: {} @@ -99,7 +99,7 @@ graphs: version: v1 handler: example.example:train runtime: python:3.8 - codeuri: null + codeurl: null image: null framework: name: adaptdl @@ -123,7 +123,7 @@ graphs: version: v1 handler: example.example:evaluate runtime: python:3.8 - codeuri: null + codeurl: null image: null framework: null config: {} diff --git a/pirlib/pir.py b/pirlib/pir.py index f12395f..9f1cc0c 100644 --- a/pirlib/pir.py +++ b/pirlib/pir.py @@ -1,6 +1,15 @@ """ +This module contains dataclasses that define the PIR (Pipeline Intermediate +Representation) format. The top-level contaner is the :obj:`Package`, which consists +of one or more :obj:`Graph` objects. Each graph is a directed acyclic graph (DAG) of +:obj:`Node` or :obj:`Subgraph` elements. Each node represents some executable procedure +with well-defined inputs and outputs, and the graph structure defines the dependencies +between those inputs and outputs. A graph can also embed subgraphs which are other +graphs in the same package. A visual example of a package is shown below. + .. image:: ../_static/img/pir-diagram.svg """ +from __future__ import annotations import copy import re @@ -12,224 +21,146 @@ @dataclass -class DataSource: +class Package: """ - This dataclass encodes a reference to the source of an intermediate value in a PIR - graph. A source can either be (1) an output of a node in the graph, (2) an output - of a subgraph of the graph, or (3) an input of the graph itself. + This dataclass encodes a package containing multiple graphs. Graphs in the same + package can embed each other as subgraphs, as long as they are not recursively + nested. - :ivar node: Name of the node in the graph if the source is the output of that node, - ``None`` otherwise. - :ivar subgraph: Name of the subgraph in the graph if the source is the output of - that subgraph, ``None`` otherwise. - :ivar output: Name of the output of a node or subgraph if the source is the output - of that node or subgraph, ``None`` otherwise. - :ivar graph_input: Name of the input of the graph if the source is the input of the - graph, ``None`` otherwise. + :ivar graphs: List of graphs in this package, must all have unique names. """ - node: Optional[str] = None - subgraph: Optional[str] = None - output: Optional[str] = None - graph_input: Optional[str] = None + graphs: List[Graph] = field(default_factory=list) + + def flatten_graph(self, graph_name: str, validate: bool = True) -> Graph: + """ + Return a graph from this package after flattening all its subgraphs. The + resulting graph should only contain nodes and no subgraphs. Each node in a + subgraph is converted into a node in the parent graph with a new node name + equal to ``.``. + + :param graph_name: Name of the graph in this package to flatten. + :param validate: Validate the resulting flattened graph. Validation may fail, + for example, if the resulting graph has nodes with conflicting names. + :return: The resulting flattened graph. + """ + graph = copy.deepcopy(find_by_name(self.graphs, graph_name)) + if graph is None: + raise ValueError(f"graph with name '{graph_name}' not found in package") + for subgraph in graph.subgraphs: + g = self.flatten_graph(subgraph.graph, validate=False) + # Add prefix to subgraph nodes names. + for n in g.nodes: + n.name = f"{subgraph.name}.{n.name}" + for i in n.inputs: + if i.source.node is not None: + i.source.node = f"{subgraph.name}.{i.source.node}" + for o in g.outputs: + if o.source.node is not None: + o.source.node = f"{subgraph.name}.{o.source.node}" + # Merge subgraph into main graph. + for node in graph.nodes: + for inp in node.inputs: + if inp.source.subgraph == subgraph.name: + for o in g.outputs: + if o.name == inp.source.output: + inp.source = o.source + for out in graph.outputs: + if out.source.subgraph == subgraph.name: + for o in g.outputs: + if o.name == out.source.output: + out.source = o.source + for n in g.nodes: + for i in n.inputs: + if i.source.graph_input is not None: + for si in subgraph.inputs: + if i.source.graph_input == si.name: + i.source = si.source + graph.nodes.extend(g.nodes) + graph.subgraphs = [] + if validate: + graph.validate() + return graph def validate(self): - _validate_fields(self) - count = sum( - [ - self.node is not None, - self.subgraph is not None, - self.graph_input is not None, - ] - ) - if count != 1: + """ + Validate this package and all graphs contained within it. A package is valid if + (1) all of its graphs are valid and have unique names, (2) all subgraphs are + valid references to another graph in the same package, and (3) there are no + recursively nested subgraphs. + + :raises ValidationError: If the package is invalid. + """ + _validate_names(self.graphs, "graph") + for graph in self.graphs: + try: + graph.validate() + except ValidationError as err: + raise ValidationError(f"graph '{graph.name}': {err}") from None + for subgraph in graph.subgraphs: + self._validate_subgraph(subgraph) + if self._is_recursive(graph, []): + raise ValidationError(f"package contains recursive subgraphs") + + def _validate_subgraph(self, subgraph): + graph = find_by_name(self.graphs, subgraph.graph) + if graph is None: raise ValidationError( - "exactly one of 'node', 'subgraph', or 'graph_input' is expected" + f"subgraph '{subgraph.name}' refers to missing graph '{subgraph.graph}'" ) - if self.node is not None or self.subgraph is not None: - if self.output is None: + for inp in subgraph.inputs: + g_inp = find_by_name(graph.inputs, inp.name) + if g_inp is None: raise ValidationError( - "'output' is required if either 'node' or 'subgraph' is provided" + f"subgraph '{subgraph.name}' input '{inp.name}' " + f"could not be found in graph '{subgraph.graph}'" + ) + if g_inp.iotype != inp.iotype: + raise ValidationError( + f"subgraph '{subgraph.name}' input '{inp.name}' iotype " + f"'{inp.iotype}' does not match graph '{subgraph.graph}' " + f"input iotype '{g_inp.iotype}'" + ) + for out in subgraph.outputs: + g_out = find_by_name(graph.outputs, out.name) + if g_out is None: + raise ValidationError( + f"subgraph '{subgraph.name}' output '{out.name}' " + f"could not be found in graph '{subgraph.graph}'" + ) + if g_out.iotype != out.iotype: + raise ValidationError( + f"subgraph '{subgraph.name}' output '{out.name}' iotype " + f"'{out.iotype}' does not match graph '{subgraph.graph}' " + f"output iotype '{g_out.iotype}'" ) - -@dataclass -class Input: - """ - This dataclass encodes a named input of a node or subgraph with a connected source. - If it is the input of a subgraph, then its name must be equal to a name of some - graph input of that subgraph. - - :ivar name: Name of the input. Must be unique among all inputs in a valid node or - subgraph. - :ivar iotype: Expected type of the input. In a valid graph, must be equal to the - iotype of the source. - :ivar source: Source of the input. Can be a graph input or a node/subgraph output. - """ - name: str - iotype: str - source: DataSource - - def validate(self): - _validate_fields(self) - try: - self.source.validate() - except ValidationError as err: - raise ValidationError(f"source: {err}") from None - - -@dataclass -class Output: - """ - This dataclass encodes a named output of a node or a subgraph. An output can be an - input source for other downstream nodes or subgraphs within the same graph, or be - an output of the graph itself. - - :ivar name: Name of the output. Must be unique among all outputs in a valid node or - subgraph. - :ivar iotype: Type of the output. - """ - name: str - iotype: str - - def validate(self): - _validate_fields(self) - - -@dataclass -class Framework: - """ - This dataclass encodes the execution framework and configuration for a node. - - :ivar name: Name of the framework used for executing a node. - :ivar config: Framework configuration for executing a node. - """ - name: str - config: Dict[str, Any] = field(default_factory=dict) - - def validate(self): - _validate_fields(self) - - -@dataclass -class Entrypoint: - """ - This dataclass encodes the entrypoint for executing a node. An entrypoint is - typically a reference to a function or procedure in code (called "handler"). - - :ivar version: The API version for the handler. - :ivar handler: Reference to the handler, format depends on the runtime. For python - runtimes, expects ``.``, where ```` is the fully - qualified name of the module containing the handler, and ```` is the - name of the handler object within that module. - :ivar runtime: Identifier of the handler's runtime, e.g. ``"python:3.8"``. - :ivar codeurl: Optional URL of the code used to run the handler. ``None`` means any - and all handler code can be found in the local environment or docker image. - :ivar image: Optional name of the Docker image used to run the handler. ``None`` - means the handler can be run in the local environment. - """ - version: str - handler: str - runtime: str - codeurl: Optional[str] = None - image: Optional[str] = None - - def validate(self): - _validate_fields(self) + def _is_recursive(self, graph: Graph, visited: List[str]): + if graph.name in visited: + return True + visited = visited + [graph.name] + for subgraph in graph.subgraphs: + if self._is_recursive(find_by_name(self.graphs, subgraph.graph), visited): + return True + return False @dataclass -class Node: +class Graph: """ - This dataclass encodes a node in a graph. A node represents a procedure that can be - executed on several inputs to produce several outputs. All node inputs must have - unique names, and all node outputs also must have unique names. - - :ivar name: Name of the node. Must be unique among all nodes in a valid graph. - :ivar entrypoint: Entrypoint to the code executed by this node. - :ivar framework: Execution framework for this node. - :ivar config: Configuration values which are passed down to the procedure executed - by this node. Can be any json or yaml serializable mapping. - :ivar inputs: Expected inputs for this node, must all have unique names. - :ivar outputs: Expected outputs for this node, must all have unique names. + This dataclass encodes a directed acyclic graph (DAG) of nodes each with well + defined inputs and outputs. The graph itself also has inputs and outputs, where its + inputs are placeholders for values to be provided when the graph is executed. Each + graph can also embed as subgraphs any other graph in the same package. + + :ivar name: Name of the graph, must be unique among all graphs in a package. + :ivar nodes: The nodes contained in the graph. Node names must be unique among all + nodes and subgraphs in the graph. + :ivar subgraphs: The subgraphs contained in the graph. Subgraph names must be unique + among all nodes and subgraphs in the graph. + :ivar inputs: The expected inputs for the graph, must all have unique names. + :ivar outputs: The outputs for the graph, must all have unique names. Each graph + output must have the same iotype as its source. """ - name: str - entrypoint: Entrypoint - framework: Optional[Framework] = None - config: Dict[str, Any] = field(default_factory=dict) - inputs: List[Input] = field(default_factory=list) - outputs: List[Output] = field(default_factory=list) - - def validate(self): - _validate_fields(self) - for inp in self.inputs: - try: - inp.validate() - except ValidationError as err: - raise ValidationError(f"input '{inp.name}': {err}") from None - _validate_names(self.inputs, "input") - for out in self.outputs: - try: - out.validate() - except ValidationError as err: - raise ValidationError(f"output '{out.name}': {err}") from None - _validate_names(self.outputs, "output") - try: - self.entrypoint.validate() - except ValidationError as err: - raise ValidationError(f"entrypoint: {err}") from None - if self.framework is not None: - try: - self.framework.validate() - except ValidationError as err: - raise ValidationError(f"framework: {err}") from None - - -@dataclass -class Subgraph: - name: str - graph: str - config: Dict[str, Any] = field(default_factory=dict) - inputs: List[Input] = field(default_factory=list) - outputs: List[Output] = field(default_factory=list) - - def validate(self): - _validate_fields(self) - for inp in self.inputs: - try: - inp.validate() - except ValidationError as err: - raise ValidationError(f"input '{inp.name}': {err}") from None - _validate_names(self.inputs, "input") - for out in self.outputs: - try: - out.validate() - except ValidationError as err: - raise ValidationError(f"output '{out.name}': {err}") from None - _validate_names(self.outputs, "output") - - -@dataclass -class GraphInput: - name: str - iotype: str - - def validate(self): - _validate_fields(self) - - -@dataclass -class GraphOutput: - name: str - iotype: str - source: DataSource - - def validate(self): - _validate_fields(self) - self.source.validate() - - -@dataclass -class Graph: name: str nodes: List[Node] = field(default_factory=list) subgraphs: List[Subgraph] = field(default_factory=list) @@ -237,19 +168,25 @@ class Graph: outputs: List[GraphOutput] = field(default_factory=list) def validate(self): + """ + Validate this graph. A graph is valid if (1) all of its nodes, subgraphs, + inputs, and outputs are valid, (2) all node/subgraph inputs and graph outputs + have valid sources, and (3) there are no cycles in the graph. + + :raises ValidationError: If the graph is invalid. + """ _validate_fields(self) for node in self.nodes: try: node.validate() except ValidationError as err: raise ValidationError(f"node '{node.name}': {err}") from None - _validate_names(self.nodes, "node") for subgraph in self.subgraphs: try: subgraph.validate() except ValidationError as err: raise ValidationError(f"subgraph '{subgraph.name}': {err}") from None - _validate_names(self.subgraphs, "subgraph") + _validate_names(self.nodes + self.subgraphs, "node or subgraph") for inp in self.inputs: try: inp.validate() @@ -279,6 +216,14 @@ def _validate_connectivity(self): raise ValidationError( f"node '{node.name}': input '{inp.name}': {err}" ) from None + for subgraph in self.subgraphs: + for inp in subgraph.inputs: + try: + self._validate_source(inp.source, inp.iotype) + except ValidationError as err: + raise ValidationError( + f"subgraph '{subgraph.name}': input '{inp.name}': {err}" + ) from None def _validate_source(self, source, iotype): if source.graph_input is not None: @@ -352,68 +297,261 @@ def _validate_acyclicity(self): @dataclass -class Package: - graphs: List[Graph] = field(default_factory=list) +class GraphInput: + """ + This dataclass encodes an input of a graph. A graph input represents a + "placeholder" for a user-provided input value, and so does not have a connected + source. - def flatten_graph(self, graph_name, validate=False): - # TODO: error checking - graph = copy.deepcopy(find_by_name(self.graphs, graph_name)) - for subgraph in graph.subgraphs: - g = self.flatten_graph(subgraph.graph) - # Add prefix to subgraph nodes names. - for n in g.nodes: - n.name = f"{subgraph.name}.{n.name}" - for i in n.inputs: - if i.source.node is not None: - i.source.node = f"{subgraph.name}.{i.source.node}" - for o in g.outputs: - if o.source.node is not None: - o.source.node = f"{subgraph.name}.{o.source.node}" - # Merge subgraph into main graph. - for node in graph.nodes: - for inp in node.inputs: - if inp.source.subgraph == subgraph.name: - for o in g.outputs: - if o.name == inp.source.output: - inp.source = o.source - for out in graph.outputs: - if out.source.subgraph == subgraph.name: - for o in g.outputs: - if o.name == out.source.output: - out.source = o.source - for n in g.nodes: - for i in n.inputs: - if i.source.graph_input is not None: - for si in subgraph.inputs: - if i.source.graph_input == si.name: - i.source = si.source - graph.nodes.extend(g.nodes) - graph.subgraphs = [] - graph.validate() - return graph + :ivar name: Name of the input, must be unique among all inputs of the graph. + :ivar iotype: Expected type of the input. + """ + name: str + iotype: str def validate(self): - _validate_names(self.graphs, "graph") - for graph in self.graphs: + _validate_fields(self) + + +@dataclass +class GraphOutput: + """ + This dataclass encodes an output of a graph. Since a graph itself does not perform + any computation, a graph output simply refers to an output of a node or subgraph, or + an input of the same graph. + + :ivar name: Name of the graph output, must be unique among all outputs of the graph. + :ivar iotype: Type of the graph output, must be equal to the type of the source. + :ivar source: The source of the graph output. + """ + name: str + iotype: str + source: DataSource + + def validate(self): + _validate_fields(self) + self.source.validate() + + +@dataclass +class Subgraph: + """ + This dataclass encodes a subgraph embedded in a graph. A subgraph can refer to any + other graph in the same package as the parent graph. Each of the subgraph's inputs + and outputs must correspond to (i.e. have the same name and iotype as) an input or + output of the embedded graph. The configs of the subgraph override the configs of + the embedded graph. + + :ivar name: Name of the subgraph. Must be unique among all nodes and subgraphs in a + valid graph. + :ivar graph: Name of the graph to be embedded as a subgraph. Must be in the same + package as the parent graph. + :ivar config: Config values that override the embedded graph's configs. + :ivar inputs: Expected inputs for this subgraph, must all have unique names. Each + input must have the same name and iotype as an input of the embedded graph. + :ivar outputs: Expected outputs for this subgraph, must all have unique names. Each + output must have the same name and iotype as an output of the embedded + graph. + """ + name: str + graph: str + config: Dict[str, Dict[str, Any]] = field(default_factory=dict) + inputs: List[Input] = field(default_factory=list) + outputs: List[Output] = field(default_factory=list) + + def validate(self): + _validate_fields(self) + for inp in self.inputs: try: - graph.validate() + inp.validate() except ValidationError as err: - raise ValidationError(f"graph '{graph.name}': {err}") from None - for subgraph in graph.subgraphs: - self._validate_subgraph(subgraph) - # TODO: check for infinitely nested subgraphs + raise ValidationError(f"input '{inp.name}': {err}") from None + _validate_names(self.inputs, "input") + for out in self.outputs: + try: + out.validate() + except ValidationError as err: + raise ValidationError(f"output '{out.name}': {err}") from None + _validate_names(self.outputs, "output") - def _validate_subgraph(self, subgraph): - graph = find_by_name(self.graphs, subgraph.graph) - if graph is None: + +@dataclass +class Node: + """ + This dataclass encodes a node in a graph. A node represents a procedure that can be + executed on several inputs to produce several outputs. All node inputs must have + unique names, and all node outputs also must have unique names. + + :ivar name: Name of the node. Must be unique among all nodes and subgraphs in a + valid graph. + :ivar entrypoint: Entrypoint to the code executed by this node. + :ivar framework: Execution framework for this node. + :ivar config: Configuration values which are passed down to the procedure executed + by this node. Can be any json or yaml serializable mapping. + :ivar inputs: Expected inputs for this node, must all have unique names. + :ivar outputs: Expected outputs for this node, must all have unique names. + """ + name: str + entrypoint: Entrypoint + framework: Optional[Framework] = None + config: Dict[str, Any] = field(default_factory=dict) + inputs: List[Input] = field(default_factory=list) + outputs: List[Output] = field(default_factory=list) + + def validate(self): + _validate_fields(self) + for inp in self.inputs: + try: + inp.validate() + except ValidationError as err: + raise ValidationError(f"input '{inp.name}': {err}") from None + _validate_names(self.inputs, "input") + for out in self.outputs: + try: + out.validate() + except ValidationError as err: + raise ValidationError(f"output '{out.name}': {err}") from None + _validate_names(self.outputs, "output") + try: + self.entrypoint.validate() + except ValidationError as err: + raise ValidationError(f"entrypoint: {err}") from None + if self.framework is not None: + try: + self.framework.validate() + except ValidationError as err: + raise ValidationError(f"framework: {err}") from None + + +@dataclass +class Input: + """ + This dataclass encodes a named input of a node or subgraph with a connected source. + If it is the input of a subgraph, then its name must be equal to a name of some + graph input of that subgraph. + + :ivar name: Name of the input. Must be unique among all inputs in a valid node or + subgraph. + :ivar iotype: Expected type of the input. In a valid graph, must be equal to the + iotype of the source. + :ivar source: Source of the input. Can be a graph input or a node/subgraph output. + """ + name: str + iotype: str + source: DataSource + + def validate(self): + _validate_fields(self) + try: + self.source.validate() + except ValidationError as err: + raise ValidationError(f"source: {err}") from None + + +@dataclass +class Output: + """ + This dataclass encodes a named output of a node or a subgraph. An output can be an + input source for other downstream nodes or subgraphs within the same graph, or be + an output of the graph itself. + + :ivar name: Name of the output. Must be unique among all outputs in a valid node or + subgraph. + :ivar iotype: Type of the output. + """ + name: str + iotype: str + + def validate(self): + _validate_fields(self) + + +@dataclass +class Framework: + """ + This dataclass encodes the execution framework and configuration for a node. + + :ivar name: Name of the framework used for executing a node. + :ivar config: Framework configuration for executing a node. + """ + name: str + config: Dict[str, Any] = field(default_factory=dict) + + def validate(self): + _validate_fields(self) + + +@dataclass +class Entrypoint: + """ + This dataclass encodes the entrypoint for executing a node. An entrypoint is + typically a reference to a function or procedure in code (called "handler"). + + :ivar version: The API version for the handler. + :ivar handler: Reference to the handler, format depends on the runtime. For python + runtimes, expects ``.``, where ```` is the fully + qualified name of the module containing the handler, and ```` is the + name of the handler object within that module. + :ivar runtime: Identifier of the handler's runtime, e.g. ``"python:3.8"``. + :ivar codeurl: Optional URL of the code used to run the handler. ``None`` means any + and all handler code can be found in the local environment or docker image. + :ivar image: Optional name of the Docker image used to run the handler. ``None`` + means the handler can be run in the local environment. + """ + version: str + handler: str + runtime: str + codeurl: Optional[str] = None + image: Optional[str] = None + + def validate(self): + _validate_fields(self) + + +@dataclass +class DataSource: + """ + This dataclass encodes a reference to the source of an intermediate value in a PIR + graph. A source can either be (1) an output of a node in the graph, (2) an output + of a subgraph of the graph, or (3) an input of the graph itself. Exactly one of + ``node``, ``subgraph``, or ``graph_input`` must be provided. + + :ivar node: Name of a node if the source is the output of that node. + :ivar subgraph: Name of a subgraph if the source is the output of that subgraph. + :ivar output: Name of the output of a node or subgraph if the source is the output + of that node or subgraph. + :ivar graph_input: Name of the graph input if the source is an input of the graph. + """ + node: Optional[str] = None + subgraph: Optional[str] = None + output: Optional[str] = None + graph_input: Optional[str] = None + + def validate(self): + _validate_fields(self) + count = sum( + [ + self.node is not None, + self.subgraph is not None, + self.graph_input is not None, + ] + ) + if count != 1: raise ValidationError( - f"subgraph '{subgraph.name}': reference to missing graph " - f"'{subgraph.graph}'" + "exactly one of 'node', 'subgraph', or 'graph_input' is expected" ) - # TODO: check subgraph inputs and outputs match graph inputs and outputs + if self.node is not None or self.subgraph is not None: + if self.output is None: + raise ValidationError( + "'output' is required if either 'node' or 'subgraph' is provided" + ) class ValidationError(ValueError): + """ + Exception raised when any part of a PIR package fails validation. + """ + def __init__(self, message): super().__init__(message)