Skip to content

Commit 648a7c4

Browse files
fengjiachunMichaelScofieldclaude
authored
feat: remove stream & add bulk insert (#23)
* refactor: minor refactor * feat: filght encoder * feat: bulk insert example * chore: remove unused code * feat: bulk stream writer * chore: no panic * feat: write record batch parallel * chore: minor refactor * feat: docs * feat: integration test * chore: method name, without_compression * chore: fix integration test * chore: intergration test CI * feat: use duration to set timeout * chore: remove drop for BulkStreamWriter * Update README.md * Update examples/README.md * fix: clippy * fix: clippy * fix: clippy * Update README.md Co-authored-by: LFC <[email protected]> * chore: timeout for wait stream next item * chore: let wait_for_all_pending can timeout * feat: make pending_requests and completed_resposne simply * feat: rm readme_test and make rust code in readme can compile * fix: by comment * chore: refactor request_id * chore: refactor wait_for_all_pending * chore: minor change * fix: make all stream operations can timeout * chore: minor change * fix: improve concurrency control in process_pending_responses 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <[email protected]> --------- Co-authored-by: LFC <[email protected]> Co-authored-by: Claude <[email protected]>
1 parent f724339 commit 648a7c4

24 files changed

+3827
-1424
lines changed

‎.github/workflows/integration-tests.yml

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
name: Integration Tests
2+
3+
on:
4+
push:
5+
branches: [ master, main, develop ]
6+
pull_request:
7+
branches: [ master, main, develop ]
8+
schedule:
9+
# Run integration tests daily at 2 AM UTC
10+
- cron: '0 2 * * *'
11+
12+
env:
13+
CARGO_TERM_COLOR: always
14+
GREPTIMEDB_TEST_ENDPOINT: localhost:4001
15+
GREPTIMEDB_TEST_DATABASE: public
16+
17+
jobs:
18+
integration-tests:
19+
runs-on: ubuntu-latest
20+
21+
steps:
22+
- name: Checkout repository
23+
uses: actions/checkout@v4
24+
25+
- name: Setup Rust toolchain
26+
uses: dtolnay/rust-toolchain@stable
27+
with:
28+
components: rustfmt, clippy
29+
30+
- name: Setup Rust cache
31+
uses: Swatinem/rust-cache@v2
32+
with:
33+
key: integration-tests
34+
35+
- name: Install Protocol Buffers compiler
36+
run: |
37+
sudo apt-get update
38+
sudo apt-get install -y protobuf-compiler
39+
protoc --version
40+
41+
- name: Run format check
42+
run: cargo fmt --all -- --check
43+
44+
- name: Run clippy
45+
run: cargo clippy --all-targets --all-features -- -D warnings
46+
47+
- name: Build project
48+
run: cargo build --release
49+
50+
- name: Run unit tests
51+
run: cargo test --lib
52+
53+
- name: Start GreptimeDB with Docker Compose
54+
run: |
55+
docker compose -f docker-compose.test.yml up -d
56+
timeout 60 bash -c 'until curl -f http://localhost:4000/health; do sleep 1; done'
57+
echo "GreptimeDB is ready!"
58+
59+
- name: Run integration tests
60+
run: cargo test --test integration --features integration-tests --verbose
61+
env:
62+
RUST_LOG: info
63+
64+
- name: Cleanup
65+
run: docker compose -f docker-compose.test.yml down -v
66+
if: always()

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
/target
22
/Cargo.lock
3+
.DS_Store

‎Cargo.toml

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,24 +6,50 @@ license = "Apache-2.0"
66
description = "A rust client for GreptimeDB gRPC protocol"
77

88
[dependencies]
9+
arrow = { version = "54.2", features = ["prettyprint"] }
10+
arrow-array = { version = "54.2", default-features = false, features = ["chrono-tz"] }
11+
arrow-flight = "54.2"
12+
arrow-ipc = { version = "54.2", default-features = false, features = ["lz4", "zstd"] }
13+
arrow-schema = { version = "54.2", features = ["serde"] }
14+
async-stream = "0.3"
15+
async-trait = "0.1"
16+
base64 = "0.22"
917
dashmap = "6.1"
1018
enum_dispatch = "0.3"
19+
flatbuffers = "24"
1120
futures = "0.3"
1221
futures-util = "0.3"
13-
greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", tag = "v0.9.0" }
22+
greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "454c52634c3bac27de10bf0d85d5533eed1cf03f" }
23+
hyper = "1.1"
24+
lazy_static = "1.4"
1425
parking_lot = "0.12"
15-
prost = "0.12"
26+
prost = { version = "0.13", features = ["no-recursion-limit"] }
1627
rand = "0.9"
28+
serde = { version = "1.0", features = ["derive"] }
29+
serde_json = { version = "1.0", features = ["float_roundtrip"] }
1730
snafu = "0.8"
18-
tokio = { version = "1", features = ["rt", "time"] }
31+
tokio = { version = "1.40", features = ["full"] }
1932
tokio-stream = { version = "0.1", features = ["net"] }
20-
tonic = { version = "0.11", features = ["tls", "tls-roots", "gzip", "zstd"] }
21-
tower = "0.4"
33+
tokio-util = { version = "0.7", features = ["io-util", "compat"] }
34+
tonic = { version = "0.12", features = ["tls", "gzip", "zstd"] }
35+
tower = "0.5"
2236
derive_builder = "0.20"
2337

2438
[build-dependencies]
25-
tonic-build = "0.9"
39+
tonic-build = "0.11"
2640

2741
[dev-dependencies]
28-
tokio = { version = "1", features = ["full"] }
42+
hyper-util = "0.1"
43+
tokio = { version = "1.40", features = ["full"] }
2944
derive-new = "0.7"
45+
46+
# Integration tests configuration
47+
[[test]]
48+
name = "integration"
49+
path = "tests/integration/integration_test.rs"
50+
harness = true
51+
required-features = ["integration-tests"]
52+
53+
[features]
54+
default = []
55+
integration-tests = []

0 commit comments

Comments
 (0)