feat: Partial support for SDK style libraries in Rust (#510)

*Issue #, if available:* #516 *Description of changes:* First cut of actual codegen support for Rust, only for SDK-style libraries. I implemented this earlier in order to generate the "benerated" code in #413. The code is not organized terribly well and definitely under-tested, but this will naturally improve as we replace beneration with actual code generation for all the TestModels. `kms-lite` and `ddb-lite` are also updated with the latest generated code, and both include more operations that turned out to be used by the MPL and so on. Known caveats: * Unions are not supported yet - `ddb-lite/src/conversions/attribute_value.rs` is manually written. * There is a lot of code duplication which we should clean up before considering Rust codegen complete. We should also factor out a lot of the conversion code generation as it will apply to local services as well. * The Smithy build plugin doesn't support the same patching mechanism that the CLI does, so instead I've restored Mikael's hack to insert lines in `implementation_from_dafny.rs`, this time to declare the additional external modules. This will work for local services as well but I've only implement the SDK case for now.
smithy-lang · Aug 15, 2024 · 9b1bf1d · 9b1bf1d
1 parent 51c93d9
commit 9b1bf1d
Show file tree

Hide file tree

Showing 186 changed files with 25,736 additions and 839 deletions.
diff --git a/.github/not-grep.toml b/.github/not-grep.toml
@@ -1,13 +1,13 @@
 [prefix]
-"**/*.smithy" = """
+"**/[!smithy-dafny-codegen-modules]/**/*.smithy" = """
 // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 """
-"**/*.java" = """
+"**/[!smithy-dafny-codegen-modules]/**/*.java" = """
 // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 """
-"**/[!/obj/]*.cs" = """
+"**/[!smithy-dafny-codegen-modules]/**/[!/obj/]*.cs" = """
 // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 """
@@ -16,15 +16,15 @@
 // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 """
-"**/*.kts" = """
+"**/[!smithy-dafny-codegen-modules]/**/*.kts" = """
 // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 """
-"**/*.mk" = """
+"**/[!smithy-dafny-codegen-modules]/**/*.mk" = """
 # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 """
-"**/Makefile" = """
+"**/[!smithy-dafny-codegen-modules]/**/Makefile" = """
 # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
-"""
+"""
diff --git a/.github/workflows/smithy-dafny-conversion.yml b/.github/workflows/smithy-dafny-conversion.yml
@@ -11,6 +11,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - uses: actions/setup-java@v3
         with:
           distribution: "corretto"

diff --git a/.github/workflows/smithy-polymorph.yml b/.github/workflows/smithy-polymorph.yml
@@ -15,6 +15,8 @@ jobs:
     runs-on: ${{matrix.os}}
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - uses: actions/setup-java@v3
         with:
           distribution: "corretto"
@@ -29,6 +31,12 @@ jobs:
           # Matching the hard-coded version for the "2023" edition for now
           dafny-version: 4.1.0
 
+      - name: Install smithy-dafny-codegen dependencies locally
+        uses: gradle/gradle-build-action@v2
+        with:
+          arguments: :codegen-client:pTML :codegen-core:pTML :rust-runtime:pTML
+          build-root-directory: smithy-dafny-codegen-modules/smithy-rs
+
       - name: Execute smithy-dafny-codegen-cli tests
         uses: gradle/gradle-build-action@v2
         with:

diff --git a/.github/workflows/test_models_dafny_verification.yml b/.github/workflows/test_models_dafny_verification.yml
@@ -44,6 +44,8 @@ jobs:
           git config --global core.longpaths true
 
       - uses: actions/checkout@v2
+        with:
+          submodules: recursive
 
       - name: Setup Dafny
         uses: dafny-lang/setup-dafny-action@v1.7.0
@@ -56,6 +58,12 @@ jobs:
           distribution: "corretto"
           java-version: "17"
 
+      - name: Install smithy-dafny-codegen dependencies locally
+        uses: gradle/gradle-build-action@v2
+        with:
+          arguments: :codegen-client:pTML :codegen-core:pTML :rust-runtime:pTML
+          build-root-directory: smithy-dafny-codegen-modules/smithy-rs
+
       - name: Install smithy-dafny-codegen locally
         uses: gradle/gradle-build-action@v2
         with:

diff --git a/.github/workflows/test_models_java_tests.yml b/.github/workflows/test_models_java_tests.yml
@@ -48,6 +48,8 @@ jobs:
           role-session-name: JavaTests
 
       - uses: actions/checkout@v3
+        with:
+          submodules: recursive
 
       - name: Setup Dafny
         uses: dafny-lang/setup-dafny-action@v1.7.0
@@ -66,6 +68,12 @@ jobs:
           arguments: publishToMavenLocal
           build-root-directory: smithy-dafny-conversion
 
+      - name: Install smithy-dafny-codegen dependencies locally
+        uses: gradle/gradle-build-action@v2
+        with:
+          arguments: :codegen-client:pTML :codegen-core:pTML :rust-runtime:pTML
+          build-root-directory: smithy-dafny-codegen-modules/smithy-rs
+
       - name: Install smithy-dafny-codegen locally
         uses: gradle/gradle-build-action@v2
         with:

diff --git a/.github/workflows/test_models_net_tests.yml b/.github/workflows/test_models_net_tests.yml
@@ -52,6 +52,8 @@ jobs:
           role-session-name: NetTests
 
       - uses: actions/checkout@v3
+        with:
+          submodules: recursive
 
       - name: Setup Dafny
         uses: dafny-lang/setup-dafny-action@v1.7.0
@@ -76,6 +78,12 @@ jobs:
           distribution: "corretto"
           java-version: "17"
 
+      - name: Install smithy-dafny-codegen dependencies locally
+        uses: gradle/gradle-build-action@v2
+        with:
+          arguments: :codegen-client:pTML :codegen-core:pTML :rust-runtime:pTML
+          build-root-directory: smithy-dafny-codegen-modules/smithy-rs
+
       - name: Install smithy-dafny-codegen locally
         uses: gradle/gradle-build-action@v2
         with:

diff --git a/.github/workflows/test_models_rust_tests.yml b/.github/workflows/test_models_rust_tests.yml
@@ -48,6 +48,8 @@ jobs:
           role-session-name: JavaTests
 
       - uses: actions/checkout@v3
+        with:
+          submodules: recursive
 
       - name: Setup Dafny (build from source)
         uses: ./.github/actions/build_dafny_from_source
@@ -67,6 +69,12 @@ jobs:
           distribution: "corretto"
           java-version: "17"
 
+      - name: Install smithy-dafny-codegen dependencies locally
+        uses: gradle/gradle-build-action@v2
+        with:
+          arguments: :codegen-client:pTML :codegen-core:pTML :rust-runtime:pTML
+          build-root-directory: smithy-dafny-codegen-modules/smithy-rs
+
       - name: Install smithy-dafny-codegen locally
         uses: gradle/gradle-build-action@v2
         with:

diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "smithy-dafny-codegen-modules/smithy-rs"]
+	path = smithy-dafny-codegen-modules/smithy-rs
+	url = git@github.com:smithy-lang/smithy-rs.git
diff --git a/SmithyDafnyMakefile.mk b/SmithyDafnyMakefile.mk
@@ -516,7 +516,8 @@ test_java:
 
 ########################## Rust targets
 
-# TODO: blah blah Rust only supports a single crate for everything
+# The Dafny Rust code generator only supports a single crate for everything,
+# so (among other consequences) we compile src and test code together.
 transpile_rust: | transpile_implementation_rust transpile_dependencies_rust
 
 transpile_implementation_rust: TARGET=rs
@@ -526,7 +527,8 @@ transpile_implementation_rust: TEST_INDEX=$(RUST_TEST_INDEX)
 # The Dafny Rust code generator is not complete yet,
 # so we want to emit code even if there are unsupported features in the input.
 transpile_implementation_rust: DAFNY_OPTIONS=-emitUncompilableCode
-# TODO:
+# The Dafny Rust code generator only supports a single crate for everything,
+# so we inline all dependencies by not passing `-library` to Dafny.
 transpile_implementation_rust: TRANSPILE_DEPENDENCIES=
 transpile_implementation_rust: STD_LIBRARY=
 transpile_implementation_rust: SRC_INDEX_TRANSPILE=$(if $(SRC_INDEX),$(SRC_INDEX),src)
@@ -536,9 +538,18 @@ transpile_implementation_rust: $(if $(TRANSPILE_TESTS_IN_RUST), transpile_test,
 transpile_dependencies_rust: LANG=rust
 transpile_dependencies_rust: transpile_dependencies
 
+_mv_implementation_rust: RUST_EXTERN_MODULE_DECLARATIONS=$(if $(AWS_SDK_CMD), \
+mod client; \
+mod conversions; \
+mod standard_library_conversions;, )
 _mv_implementation_rust:
 	mkdir -p runtimes/rust/src
-	mv implementation_from_dafny-rust/src/implementation_from_dafny.rs runtimes/rust/src/implementation_from_dafny.rs
+# Dafny-generated code assumes its output will be the main library file,
+# so we need to splice in module declarations for any extern code.
+	$(if $(AWS_SDK_CMD), \
+	python3 -c "import sys; data = sys.stdin.buffer.read(); sys.stdout.buffer.write(data.replace(b'\npub mod', b'\n$(RUST_EXTERN_MODULE_DECLARATIONS)\n\npub mod', 1) if b'\npub mod' in data else data)" \
+	  < implementation_from_dafny-rust/src/implementation_from_dafny.rs > runtimes/rust/src/implementation_from_dafny.rs, \
+	mv implementation_from_dafny-rust/src/implementation_from_dafny.rs runtimes/rust/src/implementation_from_dafny.rs)
 	rustfmt runtimes/rust/src/implementation_from_dafny.rs
 	rm -rf implementation_from_dafny-rust
 

diff --git a/TestModels/aws-sdks/ddb-lite/Makefile b/TestModels/aws-sdks/ddb-lite/Makefile
@@ -8,11 +8,13 @@
 
 CORES=2
 
+TRANSPILE_TESTS_IN_RUST=1
+
 include ../../SharedMakefile.mk
 
 
 PROJECT_SERVICES := \
-	ComAmazonawsDynamodb \
+	ComAmazonawsDynamodb
 
 SERVICE_NAMESPACE_ComAmazonawsDynamodb=com.amazonaws.dynamodb
 
@@ -38,6 +40,9 @@ _polymorph_dotnet:
 _polymorph_java:
 	$(GRADLEW) polymorphJava
 
+_polymorph_rust:
+	$(GRADLEW) polymorphRust
+
 # There is no wrapped target for aws-sdk types
 _polymorph: ;
 _polymorph_wrapped: ;

diff --git a/TestModels/aws-sdks/ddb-lite/build.gradle.kts b/TestModels/aws-sdks/ddb-lite/build.gradle.kts
@@ -74,6 +74,19 @@ tasks.register("polymorphJava") {
     }
 }
 
+tasks.register("polymorphRust") {
+    dependsOn("build")
+    doLast {
+        // if needed, specify a projection to use instead
+        // default (no projection) is "source"
+        val projectionName = "operation-subset"
+        copy {
+            from(layout.buildDirectory.dir("smithyprojections/" + project.name + "/" + projectionName + "/dafny-client-codegen/runtimes/rust/src"))
+            into("runtimes/rust/src")
+        }
+    }
+}
+
 buildscript {
     val smithyVersion: String by project