chore: merge asset canister feature from master (#4430)

* feat!: verifiable asset canister state (#4428)

* chore: update changelog

* fix: handle asset canister pagination (#4432)

* chore: update changelog (#4431)

---------

Co-authored-by: Vivienne Siffert <[email protected]>

Author: raymondk

.gitignore

@@ -2,6 +2,8 @@ result*
 e2e/assets/installed/
 e2e/tests-dfx/.bats/
 
+.cursor
+
 # Building
 build/
 target/

CHANGELOG.md

@@ -4,6 +4,10 @@
 
 # 0.30.1
 
+### feat: asset sync now prints the target asset canister state hash in `--verbose` mode
+
+If an asset canister is updated and `--verbose` is enabled, `dfx` will now print the state hash of the local assets before syncing. Calling `compute_state_hash` on the asset canister after syncing will eventually return the same hash.
+
 ### feat: support dogecoin for the local dev environment
 
 You can now launch a network with `dfx start --enable-dogeoin` to run the dogecoin
@@ -40,6 +44,22 @@ This incorporates the following executed proposals:
 - [139192](https://dashboard.internetcomputer.org/proposal/139192)
 - [139079](https://dashboard.internetcomputer.org/proposal/139079)
 
+### Frontend canister
+
+#### feat: `list` returns more info about assets
+
+Asset info now contains the fields `max_age: opt nat64;`, `headers: opt vec HeaderField;`, `allow_raw_access: opt bool;`, and ``is_aliased: opt bool;` in addition to the previously returned ones.
+
+#### feat!: `list` is now paginated
+
+`list` now returns info about up to 100 assets instead of all assets in the canister. `start` allows specifying the offset at which the list of assets should start. `length` allows specifying a smaller limit if e.g. headers are too large to return the default number of assets. The full argument to `list` is now `(record { start: opt nat; length: opt nat })`.
+
+#### feat: `compute_state_hash`
+
+The function `compute_state_hash` works similar to `compute_evidence`, but instead of computing a hash over a batch of changes, it computes a hash over the full asset canister content. This can be used to verify the integrity of assets e.g. between a live and a local deployment. (This will only work if builds are deterministic. If there are e.g. timestamps hidden in filenames then hashes will not match.)
+
+- Module hash: 51e80aa7ecbb94ba477bbc910c934794db674d9c441c3f013b8e09390facb389
+- https://github.com/dfinity/sdk/pull/4428
 
 # 0.30.0
 

e2e/tests-dfx/assetscanister.bash

@@ -1063,6 +1063,53 @@ CHERRIES" "$stdout"
   assert_match 'length = 24'
 }
 
+@test "asset sync works with more than 100 assets" {
+  install_asset assetscanister
+
+  dfx_start
+  dfx canister create --all
+  dfx build
+  dfx canister install e2e_project_frontend
+
+  # Create 150 assets to test that pagination works correctly during sync
+  for i in $(seq 1 150); do
+    echo "test content $i" > "src/e2e_project_frontend/assets/test$(printf "%03d" "$i").txt"
+  done
+
+  # Initial deploy should sync all 150 assets
+  assert_command dfx deploy -v
+  assert_match 'test001.txt.*1/1'
+  assert_match 'test100.txt.*1/1'
+  assert_match 'test150.txt.*1/1'
+
+  # Verify all assets were uploaded by checking a few across the range
+  assert_command dfx canister call --query e2e_project_frontend get '(record{key="/test001.txt";accept_encodings=vec{"identity"}})'
+  assert_match "test content 1"
+  assert_command dfx canister call --query e2e_project_frontend get '(record{key="/test100.txt";accept_encodings=vec{"identity"}})'
+  assert_match "test content 100"
+  assert_command dfx canister call --query e2e_project_frontend get '(record{key="/test150.txt";accept_encodings=vec{"identity"}})'
+  assert_match "test content 150"
+
+  # Modify only one asset
+  echo "modified content 075" > "src/e2e_project_frontend/assets/test075.txt"
+
+  # Redeploy - should only sync the modified asset
+  assert_command dfx deploy -v
+  assert_match 'test075.txt.*1/1'
+  assert_match 'test001.txt.*is already installed'
+  assert_match 'test150.txt.*is already installed'
+
+  # Verify the modified asset was updated
+  assert_command dfx canister call --query e2e_project_frontend get '(record{key="/test075.txt";accept_encodings=vec{"identity"}})'
+  assert_match "modified content 075"
+
+  # Verify other assets remain unchanged
+  assert_command dfx canister call --query e2e_project_frontend get '(record{key="/test074.txt";accept_encodings=vec{"identity"}})'
+  assert_match "test content 74"
+  assert_command dfx canister call --query e2e_project_frontend get '(record{key="/test076.txt";accept_encodings=vec{"identity"}})'
+  assert_match "test content 76"
+}
+
 @test "identifies content type" {
   install_asset assetscanister
 
@@ -2244,3 +2291,53 @@ EOF
   assert_command curl -v "http://$ID.localhost:$PORT/app.html"
   assert_match "set-cookie: $IC_ENV_COOKIE_REGEX_2"
 }
+
+@test "local state hash matches canister state hash" {
+  install_asset assetscanister
+  dfx_start
+  
+  # Create some test assets
+  echo "test file 1" > src/e2e_project_frontend/assets/test1.txt
+  echo "test file 2" > src/e2e_project_frontend/assets/test2.txt
+  mkdir -p src/e2e_project_frontend/assets/subdir
+  echo "nested file" > src/e2e_project_frontend/assets/subdir/nested.txt
+  
+  # Deploy with debug logging to capture local state hash
+  assert_command dfx deploy --verbose
+  LOCAL_HASH=$(echo "$stderr" | grep "Computed state hash of assets:" | sed 's/.*Computed state hash of assets: //')
+  
+  # Call the canister's compute_state_hash method
+  assert_command dfx canister call e2e_project_frontend compute_state_hash '()'
+  CANISTER_HASH=$(echo "$stdout" | grep -o '"[0-9a-f]*"' | tr -d '"')
+  
+  # Verify hashes
+  echo "Local hash: $LOCAL_HASH"
+  echo "Canister hash: $CANISTER_HASH"
+  assert_eq "${#LOCAL_HASH}" "64"
+  assert_eq "${#CANISTER_HASH}" "64"
+  assert_eq "$LOCAL_HASH" "$CANISTER_HASH"
+  
+  # Now modify an existing asset and add a new one
+  echo "modified test file 1" > src/e2e_project_frontend/assets/test1.txt
+  echo "brand new file" > src/e2e_project_frontend/assets/test3.txt
+  mkdir -p src/e2e_project_frontend/assets/another
+  echo "another nested file" > src/e2e_project_frontend/assets/another/deep.txt
+  
+  # Redeploy with the changes
+  assert_command dfx deploy --verbose
+  LOCAL_HASH_2=$(echo "$stderr" | grep "Computed state hash of assets:" | sed 's/.*Computed state hash of assets: //')
+  
+  # Call the canister's compute_state_hash method again
+  assert_command dfx canister call e2e_project_frontend compute_state_hash '()'
+  CANISTER_HASH_2=$(echo "$stdout" | grep -o '"[0-9a-f]*"' | tr -d '"')
+  
+  # Verify new hashes
+  echo "Local hash after changes: $LOCAL_HASH_2"
+  echo "Canister hash after changes: $CANISTER_HASH_2"
+  assert_eq "${#LOCAL_HASH_2}" "64"
+  assert_eq "${#CANISTER_HASH_2}" "64"
+  assert_eq "$LOCAL_HASH_2" "$CANISTER_HASH_2"
+  
+  # Verify the hash changed after modifications
+  assert_neq "$LOCAL_HASH" "$LOCAL_HASH_2"
+}

e2e/tests-icx-asset/icx-asset.bash

@@ -63,6 +63,31 @@ icx_asset_upload() {
   assert_match "notreally.js.*text/javascript.*identity"
 }
 
+@test "lists all assets when more than 100" {
+  # Create 150 assets to test pagination
+  for i in $(seq 1 150); do
+    echo "test content $i" > "src/e2e_project_frontend/assets/test$(printf "%03d" "$i").txt"
+  done
+  icx_asset_sync
+
+  icx_asset_list
+
+  # Verify we get assets from the first page
+  assert_match "test001.txt.*text/plain.*identity"
+  assert_match "test050.txt.*text/plain.*identity"
+  assert_match "test100.txt.*text/plain.*identity"
+  
+  # Verify we get assets from the second page (beyond first 100)
+  assert_match "test101.txt.*text/plain.*identity"
+  assert_match "test125.txt.*text/plain.*identity"
+  assert_match "test150.txt.*text/plain.*identity"
+
+  # Count total number of test*.txt assets listed
+  # shellcheck disable=SC2154
+  TEST_COUNT=$(echo "$stderr" | grep -c "test[0-9][0-9][0-9].txt")
+  assert_eq "150" "$TEST_COUNT"
+}
+
 @test "creates new files" {
   echo "new file content" >src/e2e_project_frontend/assets/new-asset.txt
   icx_asset_sync

src/canisters/frontend/ic-asset/src/batch_upload/plumbing.rs

@@ -30,7 +30,7 @@ const CONTENT_ENCODING_IDENTITY: &str = "identity";
 // Any file counts as at least 1 mb.
 const MAX_COST_SINGLE_FILE_MB: usize = 45;
 
-const MAX_CHUNK_SIZE: usize = 1_900_000;
+pub(crate) const MAX_CHUNK_SIZE: usize = 1_900_000;
 
 #[derive(Clone, Debug)]
 pub(crate) struct AssetDescriptor {

src/canisters/frontend/ic-asset/src/canister_api/methods/list.rs

@@ -1,21 +1,58 @@
 use crate::canister_api::methods::method_names::LIST;
 use crate::canister_api::types::{asset::AssetDetails, list::ListAssetsRequest};
+use candid::Nat;
 use ic_agent::AgentError;
 use ic_utils::Canister;
 use ic_utils::call::SyncCall;
 use std::collections::HashMap;
 
-pub(crate) async fn list_assets(
+/// Lists all assets in the canister. Handles pagination transparently.
+pub async fn list_assets(
     canister: &Canister<'_>,
 ) -> Result<HashMap<String, AssetDetails>, AgentError> {
-    let (entries,): (Vec<AssetDetails>,) = canister
-        .query(LIST)
-        .with_arg(ListAssetsRequest {})
-        .build()
-        .call()
-        .await?;
-
-    let assets: HashMap<_, _> = entries.into_iter().map(|d| (d.key.clone(), d)).collect();
+    let mut all_entries: Vec<AssetDetails> = Vec::new();
+    let mut start = 0u64;
+    let mut prev_page_size: Option<usize> = None;
+
+    // Fetch assets in pages until we get 0 items or fewer items than the previous page
+    loop {
+        let (entries,): (Vec<AssetDetails>,) = canister
+            .query(LIST)
+            .with_arg(ListAssetsRequest {
+                start: Some(Nat::from(start)),
+                length: None,
+            })
+            .build()
+            .call()
+            .await?;
+
+        let num_entries = entries.len();
+        if num_entries == 0 {
+            break;
+        }
+
+        // If we're on a subsequent page but got the same data as the first page,
+        // the canister doesn't support pagination and is returning all entries every time
+        if start > 0 && entries == all_entries {
+            break;
+        }
+
+        start += num_entries as u64;
+        all_entries.extend(entries);
+
+        // If we got fewer items than the previous page, we've reached the end
+        if let Some(prev_size) = prev_page_size {
+            if num_entries < prev_size {
+                break;
+            }
+        }
+        prev_page_size = Some(num_entries);
+    }
+
+    let assets: HashMap<_, _> = all_entries
+        .into_iter()
+        .map(|d| (d.key.clone(), d))
+        .collect();
 
     Ok(assets)
 }

src/canisters/frontend/ic-asset/src/canister_api/types/asset.rs

@@ -3,7 +3,7 @@ use serde::Deserialize;
 use std::collections::HashMap;
 
 /// Information about a content encoding stored for an asset.
-#[derive(CandidType, Debug, Deserialize)]
+#[derive(CandidType, Debug, Deserialize, PartialEq, Eq)]
 pub struct AssetEncodingDetails {
     /// A content encoding, such as "gzip".
     pub content_encoding: String,
@@ -14,7 +14,7 @@ pub struct AssetEncodingDetails {
 }
 
 /// Information about an asset stored in the canister.
-#[derive(CandidType, Debug, Deserialize)]
+#[derive(CandidType, Debug, Deserialize, PartialEq, Eq)]
 pub struct AssetDetails {
     /// The key identifies the asset.
     pub key: String,

src/canisters/frontend/ic-asset/src/canister_api/types/list.rs

@@ -1,5 +1,8 @@
-use candid::CandidType;
+use candid::{CandidType, Nat};
 
 /// Return a list of all assets in the canister.
 #[derive(CandidType, Debug)]
-pub struct ListAssetsRequest {}
+pub struct ListAssetsRequest {
+    pub start: Option<Nat>,
+    pub length: Option<Nat>,
+}