Compare commits
23 Commits
miner
...
miner-demo
| Author | SHA1 | Date | |
|---|---|---|---|
|
9e6bd78913 | ||
|
|
5d0902735f | ||
|
|
de6e78c5ae | ||
|
|
d2e55884a5 | ||
|
|
b72735453e | ||
|
|
c570aca3a7 | ||
|
|
76b4844eec | ||
|
|
9d7467a2f6 | ||
|
|
c60b6a6bac | ||
|
|
cbf0ac889b | ||
|
|
5e4a658b88 | ||
|
|
d72a0d4d61 | ||
|
|
68c8614792 | ||
|
|
0df293941d | ||
|
|
a58cc29991 | ||
|
|
96ff2cc531 | ||
|
|
1919982116 | ||
|
|
63dd33eb13 | ||
|
|
0b8d398751 | ||
|
|
e40bdcf138 | ||
|
|
8daa7a7e9f | ||
|
|
634f265e66 | ||
|
|
6883f0a2ba |
306
Cargo.lock
generated
306
Cargo.lock
generated
@@ -1008,7 +1008,7 @@ dependencies = [
|
||||
"fastrand 2.0.0",
|
||||
"hex",
|
||||
"http 0.2.9",
|
||||
"hyper 0.14.27",
|
||||
"hyper",
|
||||
"ring",
|
||||
"time",
|
||||
"tokio",
|
||||
@@ -1045,7 +1045,7 @@ dependencies = [
|
||||
"bytes 1.5.0",
|
||||
"fastrand 2.0.0",
|
||||
"http 0.2.9",
|
||||
"http-body 0.4.5",
|
||||
"http-body",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"tracing",
|
||||
@@ -1073,7 +1073,7 @@ dependencies = [
|
||||
"aws-types",
|
||||
"bytes 1.5.0",
|
||||
"http 0.2.9",
|
||||
"http-body 0.4.5",
|
||||
"http-body",
|
||||
"once_cell",
|
||||
"percent-encoding",
|
||||
"regex-lite",
|
||||
@@ -1201,7 +1201,7 @@ dependencies = [
|
||||
"crc32fast",
|
||||
"hex",
|
||||
"http 0.2.9",
|
||||
"http-body 0.4.5",
|
||||
"http-body",
|
||||
"md-5",
|
||||
"pin-project-lite",
|
||||
"sha1",
|
||||
@@ -1233,7 +1233,7 @@ dependencies = [
|
||||
"bytes-utils",
|
||||
"futures-core",
|
||||
"http 0.2.9",
|
||||
"http-body 0.4.5",
|
||||
"http-body",
|
||||
"once_cell",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
@@ -1272,11 +1272,11 @@ dependencies = [
|
||||
"aws-smithy-types",
|
||||
"bytes 1.5.0",
|
||||
"fastrand 2.0.0",
|
||||
"h2 0.3.21",
|
||||
"h2",
|
||||
"http 0.2.9",
|
||||
"http-body 0.4.5",
|
||||
"hyper 0.14.27",
|
||||
"hyper-rustls 0.24.2",
|
||||
"http-body",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"once_cell",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
@@ -1312,7 +1312,7 @@ dependencies = [
|
||||
"bytes-utils",
|
||||
"futures-core",
|
||||
"http 0.2.9",
|
||||
"http-body 0.4.5",
|
||||
"http-body",
|
||||
"itoa",
|
||||
"num-integer",
|
||||
"pin-project-lite",
|
||||
@@ -1362,8 +1362,8 @@ dependencies = [
|
||||
"futures-util",
|
||||
"headers",
|
||||
"http 0.2.9",
|
||||
"http-body 0.4.5",
|
||||
"hyper 0.14.27",
|
||||
"http-body",
|
||||
"hyper",
|
||||
"itoa",
|
||||
"matchit",
|
||||
"memchr",
|
||||
@@ -1376,7 +1376,7 @@ dependencies = [
|
||||
"serde_path_to_error",
|
||||
"serde_urlencoded",
|
||||
"sha1",
|
||||
"sync_wrapper 0.1.2",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-tungstenite",
|
||||
"tower",
|
||||
@@ -1394,7 +1394,7 @@ dependencies = [
|
||||
"bytes 1.5.0",
|
||||
"futures-util",
|
||||
"http 0.2.9",
|
||||
"http-body 0.4.5",
|
||||
"http-body",
|
||||
"mime",
|
||||
"rustversion",
|
||||
"tower-layer",
|
||||
@@ -2242,8 +2242,8 @@ dependencies = [
|
||||
"clickhouse-derive",
|
||||
"clickhouse-rs-cityhash-sys",
|
||||
"futures 0.3.28",
|
||||
"hyper 0.14.27",
|
||||
"hyper-tls 0.5.0",
|
||||
"hyper",
|
||||
"hyper-tls",
|
||||
"lz4",
|
||||
"sealed",
|
||||
"serde",
|
||||
@@ -2446,7 +2446,7 @@ dependencies = [
|
||||
"rand 0.8.5",
|
||||
"recent_projects",
|
||||
"release_channel",
|
||||
"reqwest 0.11.20",
|
||||
"reqwest",
|
||||
"rpc",
|
||||
"rustc-demangle",
|
||||
"scrypt",
|
||||
@@ -4982,25 +4982,6 @@ dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes 1.5.0",
|
||||
"fnv",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"http 1.0.0",
|
||||
"indexmap 2.2.6",
|
||||
"slab",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "half"
|
||||
version = "2.4.1"
|
||||
@@ -5320,29 +5301,6 @@ dependencies = [
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http-body"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643"
|
||||
dependencies = [
|
||||
"bytes 1.5.0",
|
||||
"http 1.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http-body-util"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f"
|
||||
dependencies = [
|
||||
"bytes 1.5.0",
|
||||
"futures-util",
|
||||
"http 1.0.0",
|
||||
"http-body 1.0.0",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http-range-header"
|
||||
version = "0.3.1"
|
||||
@@ -5383,9 +5341,9 @@ dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2 0.3.21",
|
||||
"h2",
|
||||
"http 0.2.9",
|
||||
"http-body 0.4.5",
|
||||
"http-body",
|
||||
"httparse",
|
||||
"httpdate",
|
||||
"itoa",
|
||||
@@ -5397,26 +5355,6 @@ dependencies = [
|
||||
"want",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "1.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d"
|
||||
dependencies = [
|
||||
"bytes 1.5.0",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"h2 0.4.5",
|
||||
"http 1.0.0",
|
||||
"http-body 1.0.0",
|
||||
"httparse",
|
||||
"itoa",
|
||||
"pin-project-lite",
|
||||
"smallvec",
|
||||
"tokio",
|
||||
"want",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-rustls"
|
||||
version = "0.24.2"
|
||||
@@ -5425,29 +5363,12 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"http 0.2.9",
|
||||
"hyper 0.14.27",
|
||||
"hyper",
|
||||
"log",
|
||||
"rustls 0.21.12",
|
||||
"rustls-native-certs",
|
||||
"tokio",
|
||||
"tokio-rustls 0.24.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-rustls"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"http 1.0.0",
|
||||
"hyper 1.3.1",
|
||||
"hyper-util",
|
||||
"rustls 0.23.10",
|
||||
"rustls-pki-types",
|
||||
"tokio",
|
||||
"tokio-rustls 0.26.0",
|
||||
"tower-service",
|
||||
"tokio-rustls",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5457,48 +5378,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
|
||||
dependencies = [
|
||||
"bytes 1.5.0",
|
||||
"hyper 0.14.27",
|
||||
"hyper",
|
||||
"native-tls",
|
||||
"tokio",
|
||||
"tokio-native-tls",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-tls"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
|
||||
dependencies = [
|
||||
"bytes 1.5.0",
|
||||
"http-body-util",
|
||||
"hyper 1.3.1",
|
||||
"hyper-util",
|
||||
"native-tls",
|
||||
"tokio",
|
||||
"tokio-native-tls",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-util"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56"
|
||||
dependencies = [
|
||||
"bytes 1.5.0",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"http 1.0.0",
|
||||
"http-body 1.0.0",
|
||||
"hyper 1.3.1",
|
||||
"pin-project-lite",
|
||||
"socket2 0.5.7",
|
||||
"tokio",
|
||||
"tower",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.57"
|
||||
@@ -6341,7 +6226,7 @@ dependencies = [
|
||||
"prost",
|
||||
"prost-build",
|
||||
"prost-types",
|
||||
"reqwest 0.11.20",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"sha2 0.10.7",
|
||||
]
|
||||
@@ -6674,15 +6559,21 @@ name = "miner"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-watch",
|
||||
"clap 4.4.4",
|
||||
"collections",
|
||||
"fs",
|
||||
"futures 0.3.28",
|
||||
"git",
|
||||
"gpui",
|
||||
"heed",
|
||||
"http 0.1.0",
|
||||
"ignore",
|
||||
"indicatif",
|
||||
"reqwest 0.12.5",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"tokenizers",
|
||||
"tokio",
|
||||
"tree-sitter",
|
||||
"tree-sitter-rust",
|
||||
]
|
||||
@@ -8821,11 +8712,11 @@ dependencies = [
|
||||
"encoding_rs",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2 0.3.21",
|
||||
"h2",
|
||||
"http 0.2.9",
|
||||
"http-body 0.4.5",
|
||||
"hyper 0.14.27",
|
||||
"hyper-tls 0.5.0",
|
||||
"http-body",
|
||||
"hyper",
|
||||
"hyper-tls",
|
||||
"ipnet",
|
||||
"js-sys",
|
||||
"log",
|
||||
@@ -8847,51 +8738,6 @@ dependencies = [
|
||||
"winreg 0.50.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.12.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37"
|
||||
dependencies = [
|
||||
"base64 0.22.0",
|
||||
"bytes 1.5.0",
|
||||
"encoding_rs",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2 0.4.5",
|
||||
"http 1.0.0",
|
||||
"http-body 1.0.0",
|
||||
"http-body-util",
|
||||
"hyper 1.3.1",
|
||||
"hyper-rustls 0.27.2",
|
||||
"hyper-tls 0.6.0",
|
||||
"hyper-util",
|
||||
"ipnet",
|
||||
"js-sys",
|
||||
"log",
|
||||
"mime",
|
||||
"native-tls",
|
||||
"once_cell",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"rustls-pemfile 2.1.2",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"sync_wrapper 1.0.1",
|
||||
"system-configuration",
|
||||
"tokio",
|
||||
"tokio-native-tls",
|
||||
"tokio-util",
|
||||
"tower-service",
|
||||
"url",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"wasm-streams",
|
||||
"web-sys",
|
||||
"winreg 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "resvg"
|
||||
version = "0.41.0"
|
||||
@@ -9289,19 +9135,6 @@ dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"rustls-pki-types",
|
||||
"rustls-webpki 0.102.4",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-native-certs"
|
||||
version = "0.6.3"
|
||||
@@ -9309,7 +9142,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00"
|
||||
dependencies = [
|
||||
"openssl-probe",
|
||||
"rustls-pemfile 1.0.3",
|
||||
"rustls-pemfile",
|
||||
"schannel",
|
||||
"security-framework",
|
||||
]
|
||||
@@ -9323,16 +9156,6 @@ dependencies = [
|
||||
"base64 0.21.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-pemfile"
|
||||
version = "2.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d"
|
||||
dependencies = [
|
||||
"base64 0.22.0",
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-pki-types"
|
||||
version = "1.7.0"
|
||||
@@ -10292,7 +10115,7 @@ dependencies = [
|
||||
"percent-encoding",
|
||||
"rust_decimal",
|
||||
"rustls 0.21.12",
|
||||
"rustls-pemfile 1.0.3",
|
||||
"rustls-pemfile",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.7",
|
||||
@@ -10772,12 +10595,6 @@ version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
|
||||
|
||||
[[package]]
|
||||
name = "sync_wrapper"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
|
||||
|
||||
[[package]]
|
||||
name = "synchronoise"
|
||||
version = "1.0.1"
|
||||
@@ -10811,27 +10628,6 @@ dependencies = [
|
||||
"windows 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "system-configuration"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"core-foundation",
|
||||
"system-configuration-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "system-configuration-sys"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
|
||||
dependencies = [
|
||||
"core-foundation-sys 0.8.6",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "system-interface"
|
||||
version = "0.27.1"
|
||||
@@ -11403,17 +11199,6 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-rustls"
|
||||
version = "0.26.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
|
||||
dependencies = [
|
||||
"rustls 0.23.10",
|
||||
"rustls-pki-types",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-stream"
|
||||
version = "0.1.14"
|
||||
@@ -11558,7 +11343,7 @@ dependencies = [
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http 0.2.9",
|
||||
"http-body 0.4.5",
|
||||
"http-body",
|
||||
"http-range-header",
|
||||
"pin-project-lite",
|
||||
"tower-layer",
|
||||
@@ -11576,7 +11361,7 @@ dependencies = [
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http 0.2.9",
|
||||
"http-body 0.4.5",
|
||||
"http-body",
|
||||
"http-range-header",
|
||||
"pin-project-lite",
|
||||
"tower-layer",
|
||||
@@ -12500,19 +12285,6 @@ dependencies = [
|
||||
"wasmparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-streams"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasmparser"
|
||||
version = "0.201.0"
|
||||
|
||||
@@ -9,16 +9,34 @@ license = "GPL-3.0-or-later"
|
||||
name = "miner"
|
||||
path = "src/miner.rs"
|
||||
|
||||
[features]
|
||||
test-support = [
|
||||
"collections/test-support",
|
||||
"fs/test-support",
|
||||
"gpui/test-support",
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
async-watch.workspace = true
|
||||
collections.workspace = true
|
||||
clap.workspace = true
|
||||
futures.workspace = true
|
||||
fs.workspace = true
|
||||
git.workspace = true
|
||||
gpui.workspace = true
|
||||
heed.workspace = true
|
||||
http.workspace = true
|
||||
ignore.workspace = true
|
||||
indicatif = "0.17.8"
|
||||
reqwest = { version = "0.12.5", features = ["json", "stream"] }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
tree-sitter.workspace = true
|
||||
tree-sitter-rust.workspace = true
|
||||
tokenizers = { version = "0.19.1", features = ["http"] }
|
||||
tokio.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
collections = { workspace = true, features = ["test-support"] }
|
||||
fs = { workspace = true, features = ["test-support"] }
|
||||
gpui = { workspace = true, features = ["test-support"] }
|
||||
tempfile.workspace = true
|
||||
|
||||
80
crates/miner/src/database.rs
Normal file
80
crates/miner/src/database.rs
Normal file
@@ -0,0 +1,80 @@
|
||||
use anyhow::{anyhow, Result};
|
||||
use futures::{
|
||||
channel::{mpsc, oneshot},
|
||||
SinkExt, StreamExt,
|
||||
};
|
||||
use gpui::BackgroundExecutor;
|
||||
use heed::{
|
||||
types::{SerdeJson, Str},
|
||||
Database as HeedDatabase, EnvOpenOptions, RwTxn,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{path::Path, time::SystemTime};
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct CachedSummary {
|
||||
pub summary: String,
|
||||
pub mtime: SystemTime,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Database {
|
||||
tx: mpsc::Sender<Box<dyn FnOnce(&HeedDatabase<Str, SerdeJson<CachedSummary>>, RwTxn) + Send>>,
|
||||
}
|
||||
|
||||
impl Database {
|
||||
pub async fn new(db_path: &Path, root: &Path, executor: &BackgroundExecutor) -> Result<Self> {
|
||||
std::fs::create_dir_all(&db_path)?;
|
||||
let env = unsafe {
|
||||
EnvOpenOptions::new()
|
||||
.map_size(1024 * 1024 * 1024)
|
||||
.max_dbs(3000)
|
||||
.open(db_path)?
|
||||
};
|
||||
let mut wtxn = env.write_txn()?;
|
||||
let db_name = format!("summaries_{}", root.to_string_lossy());
|
||||
let db: HeedDatabase<Str, SerdeJson<CachedSummary>> =
|
||||
env.create_database(&mut wtxn, Some(&db_name))?;
|
||||
wtxn.commit()?;
|
||||
|
||||
let (tx, mut rx) = mpsc::channel::<
|
||||
Box<dyn FnOnce(&HeedDatabase<Str, SerdeJson<CachedSummary>>, RwTxn) + Send>,
|
||||
>(100);
|
||||
|
||||
executor
|
||||
.spawn(async move {
|
||||
while let Some(f) = rx.next().await {
|
||||
let wtxn = env.write_txn().unwrap();
|
||||
f(&db, wtxn);
|
||||
}
|
||||
})
|
||||
.detach();
|
||||
|
||||
Ok(Self { tx })
|
||||
}
|
||||
|
||||
pub async fn transact<F, T>(&self, f: F) -> Result<T>
|
||||
where
|
||||
F: FnOnce(&HeedDatabase<Str, SerdeJson<CachedSummary>>, &mut RwTxn) -> Result<T>
|
||||
+ Send
|
||||
+ 'static,
|
||||
T: 'static + Send,
|
||||
{
|
||||
let (tx, rx) = oneshot::channel();
|
||||
self.tx
|
||||
.clone()
|
||||
.send(Box::new(move |db, mut txn| {
|
||||
let result = f(db, &mut txn);
|
||||
if result.is_ok() {
|
||||
if let Err(e) = txn.commit() {
|
||||
let _ = tx.send(Err(anyhow::Error::from(e)));
|
||||
return;
|
||||
}
|
||||
}
|
||||
let _ = tx.send(result);
|
||||
}))
|
||||
.await
|
||||
.map_err(|_| anyhow!("database closed"))?;
|
||||
Ok(rx.await.map_err(|_| anyhow!("transaction failed"))??)
|
||||
}
|
||||
}
|
||||
131
crates/miner/src/huggingface.rs
Normal file
131
crates/miner/src/huggingface.rs
Normal file
@@ -0,0 +1,131 @@
|
||||
use crate::{LanguageModel, Message};
|
||||
use anyhow::{anyhow, Result};
|
||||
use futures::{
|
||||
channel::mpsc, future::BoxFuture, io::BufReader, AsyncBufReadExt, AsyncReadExt, FutureExt,
|
||||
SinkExt, StreamExt,
|
||||
};
|
||||
use gpui::BackgroundExecutor;
|
||||
use http::HttpClient;
|
||||
use serde::Deserialize;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct HuggingFaceClient {
|
||||
client: Arc<dyn HttpClient>,
|
||||
endpoint: String,
|
||||
api_key: String,
|
||||
background_executor: BackgroundExecutor,
|
||||
}
|
||||
|
||||
impl HuggingFaceClient {
|
||||
pub fn new(endpoint: String, api_key: String, background_executor: BackgroundExecutor) -> Self {
|
||||
Self {
|
||||
client: http::client(None),
|
||||
endpoint,
|
||||
api_key,
|
||||
background_executor,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl LanguageModel for HuggingFaceClient {
|
||||
fn stream_completion(
|
||||
&self,
|
||||
messages: Vec<Message>,
|
||||
) -> BoxFuture<Result<mpsc::Receiver<String>>> {
|
||||
async move {
|
||||
let (mut tx, rx) = mpsc::channel(100);
|
||||
|
||||
let mut inputs = messages
|
||||
.iter()
|
||||
.map(|msg| format!("<|im_start|>{}\n{}<|im_end|>", msg.role, msg.content))
|
||||
.collect::<Vec<String>>()
|
||||
.join("\n");
|
||||
inputs.push_str("<|im_end|>");
|
||||
inputs.push_str("<|im_start|>assistant\n");
|
||||
|
||||
let request = serde_json::json!({
|
||||
"inputs": inputs,
|
||||
"stream": true,
|
||||
"max_tokens": 2048
|
||||
});
|
||||
|
||||
let request = http::Request::builder()
|
||||
.method(http::Method::POST)
|
||||
.uri(&self.endpoint)
|
||||
.header("Authorization", format!("Bearer {}", self.api_key))
|
||||
.header("Content-Type", "application/json")
|
||||
.body(http::AsyncBody::from(serde_json::to_vec(&request)?))?;
|
||||
|
||||
let mut response = self.client.send(request).await?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
let mut body = Vec::new();
|
||||
response.body_mut().read_to_end(&mut body).await?;
|
||||
let body_str = std::str::from_utf8(&body)?;
|
||||
return Err(anyhow!(
|
||||
"Failed to connect to API: {} {}",
|
||||
response.status(),
|
||||
body_str
|
||||
));
|
||||
}
|
||||
|
||||
let reader = BufReader::new(response.into_body());
|
||||
let stream = reader.lines().filter_map(|line| async move {
|
||||
match line {
|
||||
Ok(line) => {
|
||||
let line = line.strip_prefix("data: ")?;
|
||||
match serde_json::from_str::<StreamOutput>(line) {
|
||||
Ok(output) => {
|
||||
if !output.token.special {
|
||||
Some(Ok(output.token.text))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Err(error) => Some(Err(anyhow!(error))),
|
||||
}
|
||||
}
|
||||
Err(error) => Some(Err(anyhow!(error))),
|
||||
}
|
||||
});
|
||||
|
||||
self.background_executor
|
||||
.spawn(async move {
|
||||
futures::pin_mut!(stream);
|
||||
while let Some(result) = stream.next().await {
|
||||
match result {
|
||||
Ok(text) => {
|
||||
if tx.send(text).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Error in stream: {:?}", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.detach();
|
||||
|
||||
Ok(rx)
|
||||
}
|
||||
.boxed()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct StreamOutput {
|
||||
index: u32,
|
||||
token: Token,
|
||||
generated_text: Option<String>,
|
||||
details: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Token {
|
||||
id: u32,
|
||||
text: String,
|
||||
logprob: f64,
|
||||
special: bool,
|
||||
}
|
||||
16
crates/miner/src/language_model.rs
Normal file
16
crates/miner/src/language_model.rs
Normal file
@@ -0,0 +1,16 @@
|
||||
use anyhow::Result;
|
||||
use futures::{channel::mpsc, future::BoxFuture};
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct Message {
|
||||
pub role: String,
|
||||
pub content: String,
|
||||
}
|
||||
|
||||
pub trait LanguageModel: Send + Sync {
|
||||
fn stream_completion(
|
||||
&self,
|
||||
messages: Vec<Message>,
|
||||
) -> BoxFuture<Result<mpsc::Receiver<String>>>;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
105
crates/miner/src/ollama.rs
Normal file
105
crates/miner/src/ollama.rs
Normal file
@@ -0,0 +1,105 @@
|
||||
#![allow(unused)]
|
||||
|
||||
use crate::{BackgroundExecutor, LanguageModel, Message};
|
||||
use anyhow::{anyhow, Result};
|
||||
use futures::{
|
||||
channel::mpsc, future::BoxFuture, io::BufReader, AsyncBufReadExt, AsyncReadExt, FutureExt,
|
||||
SinkExt, StreamExt,
|
||||
};
|
||||
use http::{AsyncBody, HttpClient, Method, Request as HttpRequest};
|
||||
use serde::Deserialize;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct OllamaClient {
|
||||
client: Arc<dyn HttpClient>,
|
||||
base_url: String,
|
||||
model: String,
|
||||
executor: BackgroundExecutor,
|
||||
}
|
||||
|
||||
impl OllamaClient {
|
||||
pub fn new(base_url: String, model: String, executor: BackgroundExecutor) -> Self {
|
||||
Self {
|
||||
client: http::client(None),
|
||||
base_url,
|
||||
model,
|
||||
executor,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl LanguageModel for OllamaClient {
|
||||
fn stream_completion(
|
||||
&self,
|
||||
messages: Vec<Message>,
|
||||
) -> BoxFuture<Result<mpsc::Receiver<String>>> {
|
||||
async move {
|
||||
let (mut tx, rx) = mpsc::channel(100);
|
||||
|
||||
let request = serde_json::json!({
|
||||
"model": &self.model,
|
||||
"messages": messages,
|
||||
"stream": true,
|
||||
});
|
||||
|
||||
let uri = format!("{}/api/chat", self.base_url);
|
||||
let request = HttpRequest::builder()
|
||||
.method(Method::POST)
|
||||
.uri(uri)
|
||||
.header("Content-Type", "application/json")
|
||||
.body(AsyncBody::from(serde_json::to_vec(&request)?))?;
|
||||
|
||||
let mut response = self.client.send(request).await?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
let mut body = Vec::new();
|
||||
response.body_mut().read_to_end(&mut body).await?;
|
||||
let body_str = std::str::from_utf8(&body)?;
|
||||
return Err(anyhow!(
|
||||
"Failed to connect to API: {} {}",
|
||||
response.status(),
|
||||
body_str
|
||||
));
|
||||
}
|
||||
|
||||
let reader = BufReader::new(response.into_body());
|
||||
let stream = reader.lines().filter_map(|line| async move {
|
||||
match line {
|
||||
Ok(line) => match serde_json::from_str::<serde_json::Value>(&line) {
|
||||
Ok(response) => {
|
||||
if let Some(content) = response["message"]["content"].as_str() {
|
||||
Some(Ok(content.to_string()))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Err(error) => Some(Err(anyhow!(error))),
|
||||
},
|
||||
Err(error) => Some(Err(anyhow!(error))),
|
||||
}
|
||||
});
|
||||
|
||||
self.executor
|
||||
.spawn(async move {
|
||||
futures::pin_mut!(stream);
|
||||
while let Some(result) = stream.next().await {
|
||||
match result {
|
||||
Ok(text) => {
|
||||
if tx.send(text).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Error in stream: {:?}", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.detach();
|
||||
|
||||
Ok(rx)
|
||||
}
|
||||
.boxed()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user